From 3f2cd2df74aea9266ee2b5bc2f486a9ef8c1084f Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Tue, 25 Jun 2024 11:43:58 +0000 Subject: [PATCH 01/18] added backend (entity management) --- backend/src/api/endpoints/entity.py | 129 +++++++++ backend/src/api/endpoints/project.py | 37 +++ backend/src/app/core/data/crud/__init__.py | 4 + backend/src/app/core/data/crud/entity.py | 254 ++++++++++++++++++ .../src/app/core/data/crud/span_annotation.py | 34 +++ .../core/data/crud/span_text_entity_link.py | 55 ++++ backend/src/app/core/data/dto/entity.py | 68 +++++ .../core/data/dto/span_text_entity_link.py | 26 ++ backend/src/app/core/data/orm/entity.py | 41 +++ .../core/data/orm/span_text_entity_link.py | 19 ++ backend/src/app/core/db/import_all_orms.py | 2 + backend/src/main.py | 2 + 12 files changed, 671 insertions(+) create mode 100644 backend/src/api/endpoints/entity.py create mode 100644 backend/src/app/core/data/crud/entity.py create mode 100644 backend/src/app/core/data/crud/span_text_entity_link.py create mode 100644 backend/src/app/core/data/dto/entity.py create mode 100644 backend/src/app/core/data/dto/span_text_entity_link.py create mode 100644 backend/src/app/core/data/orm/entity.py create mode 100644 backend/src/app/core/data/orm/span_text_entity_link.py diff --git a/backend/src/api/endpoints/entity.py b/backend/src/api/endpoints/entity.py new file mode 100644 index 000000000..6591ddd29 --- /dev/null +++ b/backend/src/api/endpoints/entity.py @@ -0,0 +1,129 @@ +from typing import List + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from api.dependencies import get_current_user, get_db_session +from api.util import get_object_memo_for_user, get_object_memos +from api.validation import Validate +from app.core.authorization.authz_user import AuthzUser +from app.core.data.crud import Crud +from app.core.data.crud.entity import crud_entity +from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link +from app.core.data.dto.entity import ( + EntityCreate, + EntityMerge, + EntityRead, + EntityResolve, + EntityUpdate, +) +from app.core.data.dto.span_text_entity_link import SpanTextEntityLinkCreate + +router = APIRouter( + prefix="/entity", dependencies=[Depends(get_current_user)], tags=["entity"] +) + + +@router.put( + "", + response_model=EntityRead, + summary="Creates a new Entity and returns it with the generated ID.", +) +def create_new_entity( + *, + db: Session = Depends(get_db_session), + entity: EntityCreate, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + authz_user.assert_in_project(entity.project_id) + + db_obj = crud_entity.create(db=db, create_dto=entity) + + return EntityRead.model_validate(db_obj) + + +@router.get( + "/{entity_id}", + response_model=EntityRead, + summary="Returns the Entity with the given ID.", +) +def get_by_id( + *, + db: Session = Depends(get_db_session), + entity_id: int, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + authz_user.assert_in_same_project_as(Crud.ENTITY, entity_id) + db_obj = crud_entity.read(db=db, id=entity_id) + return EntityRead.model_validate(db_obj) + + +@router.patch( + "/{entity_id}", + response_model=EntityRead, + summary="Updates the Entity with the given ID.", +) +def update_by_id( + *, + db: Session = Depends(get_db_session), + entity_id: int, + entity: EntityUpdate, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + authz_user.assert_in_same_project_as(Crud.ENTITY, entity_id) + + db_obj = crud_entity.update(db=db, id=entity_id, update_dto=entity) + return EntityRead.model_validate(db_obj) + + +# add merge endpoint +@router.put( + "/merge", + response_model=EntityRead, + summary="Merges entities with given IDs.", +) +def merge_entities( + *, + db: Session = Depends(get_db_session), + entity_merge: EntityMerge, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + print("merge_entities") + authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_merge.entity_ids) + db_obj = crud_entity.merge(db=db, merge_dto=entity_merge) + return EntityRead.model_validate(db_obj) + + +# add resolve endpoint +@router.put( + "/resolve", + response_model=List[EntityRead], + summary="Resolve entities with given IDs.", +) +def resolve_entities( + *, + db: Session = Depends(get_db_session), + entity_resolve: EntityResolve, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + print("resolve_entities") + authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_resolve.entity_ids) + db_objs = crud_entity.resolve(db=db, resolve_dto=entity_resolve) + return [EntityRead.model_validate(db_obj) for db_obj in db_objs] + + +@router.delete( + "/{entity_id}", + response_model=EntityRead, + summary="Deletes the Entity with the given ID.", +) +def delete_by_id( + *, + db: Session = Depends(get_db_session), + entity_id: int, + authz_user: AuthzUser = Depends(), +) -> EntityRead: + authz_user.assert_in_same_project_as(Crud.ENTITY, entity_id) + + db_obj = crud_entity.remove(db=db, id=entity_id) + return EntityRead.model_validate(db_obj) diff --git a/backend/src/api/endpoints/project.py b/backend/src/api/endpoints/project.py index 7ca56577d..b44011671 100644 --- a/backend/src/api/endpoints/project.py +++ b/backend/src/api/endpoints/project.py @@ -15,6 +15,7 @@ from app.core.data.crud.code import crud_code from app.core.data.crud.crud_base import NoSuchElementError from app.core.data.crud.document_tag import crud_document_tag +from app.core.data.crud.entity import crud_entity from app.core.data.crud.memo import crud_memo from app.core.data.crud.project import crud_project from app.core.data.crud.project_metadata import crud_project_meta @@ -22,6 +23,7 @@ from app.core.data.dto.action import ActionQueryParameters, ActionRead from app.core.data.dto.code import CodeRead from app.core.data.dto.document_tag import DocumentTagRead +from app.core.data.dto.entity import EntityRead from app.core.data.dto.memo import AttachedObjectType, MemoCreate, MemoInDB, MemoRead from app.core.data.dto.preprocessing_job import PreprocessingJobRead from app.core.data.dto.project import ProjectCreate, ProjectRead, ProjectUpdate @@ -530,3 +532,38 @@ def find_duplicate_text_sdocs( return DuplicateFinderService().find_duplicate_text_sdocs( project_id=proj_id, max_different_words=max_different_words ) + + +@router.get( + "/{proj_id}/entity", + response_model=List[EntityRead], + summary="Returns all Entities of the Project with the given ID", +) +def get_project_entities( + *, + proj_id: int, + db: Session = Depends(get_db_session), + authz_user: AuthzUser = Depends(), +) -> List[EntityRead]: + authz_user.assert_in_project(proj_id) + + result = crud_entity.read_by_project(db=db, proj_id=proj_id) + result = [EntityRead.model_validate(entity) for entity in result] + result.sort(key=lambda c: c.id) + return result + + +@router.delete( + "/{proj_id}/entity", + response_model=List[int], + summary="Removes all Entities of the Project with the given ID if it exists", +) +def delete_project_entity( + *, + proj_id: int, + db: Session = Depends(get_db_session), + authz_user: AuthzUser = Depends(), +) -> List[int]: + authz_user.assert_in_project(proj_id) + + return crud_entity.remove_by_project(db=db, proj_id=proj_id) diff --git a/backend/src/app/core/data/crud/__init__.py b/backend/src/app/core/data/crud/__init__.py index 81c175185..947d8d8fb 100644 --- a/backend/src/app/core/data/crud/__init__.py +++ b/backend/src/app/core/data/crud/__init__.py @@ -8,6 +8,7 @@ from app.core.data.crud.concept_over_time_analysis import crud_cota from app.core.data.crud.current_code import crud_current_code from app.core.data.crud.document_tag import crud_document_tag +from app.core.data.crud.entity import crud_entity from app.core.data.crud.memo import crud_memo from app.core.data.crud.object_handle import crud_object_handle from app.core.data.crud.preprocessing_job import crud_prepro_job @@ -21,6 +22,7 @@ from app.core.data.crud.span_annotation import crud_span_anno from app.core.data.crud.span_group import crud_span_group from app.core.data.crud.span_text import crud_span_text +from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link from app.core.data.crud.timeline_analysis import crud_timeline_analysis from app.core.data.crud.user import crud_user from app.core.data.crud.whiteboard import crud_whiteboard @@ -51,3 +53,5 @@ class Crud(Enum): COTA_ANALYSIS = crud_cota USER = crud_user WHITEBOARD = crud_whiteboard + ENTITY = crud_entity + SPAN_TEXT_ENTITY_LINK = crud_span_text_entity_link diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py new file mode 100644 index 000000000..06321f24c --- /dev/null +++ b/backend/src/app/core/data/crud/entity.py @@ -0,0 +1,254 @@ +from typing import Any, Dict, List, Optional + +import srsly +from fastapi.encoders import jsonable_encoder +from sqlalchemy.orm import Session + +from app.core.data.crud.crud_base import CRUDBase +from app.core.data.crud.span_text import crud_span_text +from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link +from app.core.data.dto.action import ActionType +from app.core.data.dto.entity import ( + EntityCreate, + EntityMark, + EntityMerge, + EntityRead, + EntityResolve, + EntityUpdate, +) +from app.core.data.dto.span_text_entity_link import ( + SpanTextEntityLinkCreate, + SpanTextEntityLinkUpdate, +) +from app.core.data.orm.entity import EntityORM +from config import conf + + +class CRUDEntity(CRUDBase[EntityORM, EntityCreate, EntityUpdate]): + def create(self, db: Session, *, create_dto: EntityCreate) -> EntityORM: + dto_obj_data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) + db_obj = self.model(**dto_obj_data) + db.add(db_obj) + db.flush() + for span_text_id in create_dto.span_text_ids: + possible_link = ( + crud_span_text_entity_link.read_by_span_text_id_and_project_id( + db=db, span_text_id=span_text_id, project_id=create_dto.project_id + ) + ) + if possible_link is not None: + link = crud_span_text_entity_link.update( + db=db, + id=possible_link.id, + update_dto=SpanTextEntityLinkUpdate( + linked_entity_id=db_obj.id, + linked_span_text_id=possible_link.linked_span_text_id, + ), + ) + else: + link = crud_span_text_entity_link.create( + db=db, + create_dto=SpanTextEntityLinkCreate( + linked_entity_id=db_obj.id, linked_span_text_id=span_text_id + ), + ) + db.add(link) + db.commit() + db.refresh(db_obj) + return db_obj + + def create_multi( + self, db: Session, *, create_dtos: List[EntityCreate] + ) -> List[EntityORM]: + db_objs = [] + for create_dto in create_dtos: + dto_obj_data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) + db_obj = self.model(**dto_obj_data) + db.add(db_obj) + db_objs.append(db_obj) + db.flush() + + for db_obj, create_dto in zip(db_objs, create_dtos): + for span_text_id in create_dto.span_text_ids: + possible_link = ( + crud_span_text_entity_link.read_by_span_text_id_and_project_id( + db=db, + span_text_id=span_text_id, + project_id=create_dto.project_id, + ) + ) + if possible_link is not None: + link = crud_span_text_entity_link.update( + db=db, + id=possible_link.id, + update_dto=SpanTextEntityLinkUpdate( + linked_entity_id=db_obj.id, + linked_span_text_id=possible_link.linked_span_text_id, + ), + ) + else: + link = crud_span_text_entity_link.create( + db=db, + create_dto=SpanTextEntityLinkCreate( + linked_entity_id=db_obj.id, linked_span_text_id=span_text_id + ), + ) + db.add(link) + + db.commit() + return db_objs + + def mark(self, db: Session, *, mark_dto: EntityMark) -> EntityORM | None: + span_text_link = crud_span_text_entity_link.read_by_span_text_id_and_project_id( + db=db, span_text_id=mark_dto.span_text_id, project_id=mark_dto.project_id + ) + if span_text_link is None: + span_text = crud_span_text.read(db=db, id=mark_dto.span_text_id) + entity_orm = self.create( + db=db, + create_dto=EntityCreate( + name=span_text.text, + project_id=mark_dto.project_id, + span_text_ids=[mark_dto.span_text_id], + ), + ) + return entity_orm + return self.read(db=db, id=span_text_link.linked_entity_id) + + def mark_multi( + self, db: Session, *, mark_dtos: List[EntityMark] + ) -> List[EntityORM]: + to_update = {} + for entity_mark in mark_dtos: + span_text_link = ( + crud_span_text_entity_link.read_by_span_text_id_and_project_id( + db=db, + span_text_id=entity_mark.span_text_id, + project_id=entity_mark.project_id, + ) + ) + if span_text_link is None and entity_mark.span_text_id not in to_update: + span_text = crud_span_text.read(db=db, id=entity_mark.span_text_id) + to_update[entity_mark.span_text_id] = EntityCreate( + name=span_text.text, + project_id=entity_mark.project_id, + span_text_ids=[entity_mark.span_text_id], + ) + return self.create_multi(db=db, create_dtos=to_update.values()) + + def merge(self, db: Session, *, merge_dto: EntityMerge) -> EntityORM | None: + project_id = merge_dto.project_id + entities_to_delete = merge_dto.entity_ids[:] + span_text_ids = merge_dto.spantext_ids[:] + + for span_text_id in merge_dto.spantext_ids: + entity = self.read_by_span_text_id_and_project_id( + db=db, span_text_id=span_text_id, project_id=project_id + ) + assert ( + entity is not None + ), "SpanText given does not belong to an entity withing the current Project\nproject_id: {project_id}\nspan_text: {span_text}" + if len(entity.span_texts) <= 1: + entities_to_delete.append(entity.id) + + for entity_id in merge_dto.entity_ids: + entity = self.read(db=db, id=entity_id) + if entity is not None: + assert ( + entity.project_id == project_id + ), "Entity given is not in current Project\nproject_id: {project_id}\nentity: {entity}" + for span_text in entity.span_texts: + span_text_ids.append(span_text.id) + new_entity = self.create( + db=db, + create_dto=EntityCreate( + name=merge_dto.name, + project_id=project_id, + span_text_ids=span_text_ids, + ), + ) + self.remove_multi(db=db, ids=entities_to_delete) + return new_entity + + def resolve(self, db: Session, *, resolve_dto: EntityResolve) -> EntityORM | None: + entities_to_create = [] + entities_to_remove = [] + project_id = resolve_dto.project_id + for span_text_id in resolve_dto.spantext_ids: + entity = self.read_by_span_text_id_and_project_id( + db=db, span_text_id=span_text_id, project_id=project_id + ) + span_text = crud_span_text.read(db=db, id=span_text_id) + assert ( + entity is not None + ), "SpanText given does not belong to an entity withing the current Project\nproject_id: {project_id}\nspan_text: {span_text}" + entities_to_create.append( + EntityCreate( + name=span_text.text, + project_id=project_id, + span_text_ids=[span_text_id], + ) + ) + if len(entity.span_texts) <= 1: + entities_to_remove.append(entity.id) + for entity_id in resolve_dto.entity_ids: + entity = self.read(db=db, id=entity_id) + assert ( + entity.project_id == project_id + ), "Entity given is not in current Project\nproject_id: {project_id}\nentity: {entity}" + entities_to_remove.append(entity_id) + for span_text in entity.span_texts: + entities_to_create.append( + EntityCreate( + name=span_text.text, + project_id=project_id, + span_text_ids=[span_text.id], + ) + ) + self.remove_multi(db=db, ids=entities_to_remove) + new_entities = self.create_multi(db=db, create_dtos=entities_to_create) + + return new_entities + + def update( + self, db: Session, *, id: int, update_dto: EntityUpdate + ) -> EntityORM | None: + return super().update(db, id=id, update_dto=update_dto) + + def read_by_name_and_project( + self, db: Session, entity_name: str, proj_id: int + ) -> Optional[EntityORM]: + return ( + db.query(self.model) + .filter(self.model.name == entity_name, self.model.project_id == proj_id) + .first() + ) + + def read_by_span_text_id_and_project_id( + self, db: Session, *, span_text_id: int, project_id: int + ) -> EntityORM: + span_text_link = crud_span_text_entity_link.read_by_span_text_id_and_project_id( + db=db, span_text_id=span_text_id, project_id=project_id + ) + return self.read(db=db, id=span_text_link.linked_entity_id) + + def read_by_project(self, db: Session, proj_id: int) -> List[EntityORM]: + return db.query(self.model).filter(self.model.project_id == proj_id).all() + + def read_by_id(self, db: Session, entity_id: int) -> Optional[EntityORM]: + return db.query(self.model).filter(self.model.id == entity_id).first() + + def remove_by_project(self, db: Session, *, proj_id: int) -> List[EntityORM]: + query = db.query(self.model).filter(self.model.project_id == proj_id) + removed_orms = query.all() + ids = [removed_orm.id for removed_orm in removed_orms] + self.remove_multi(db=db, ids=ids) + return ids + + def remove_multi(self, db: Session, *, ids: List[int]) -> bool: + for id in ids: + self.remove(db=db, id=id) + return True + + +crud_entity = CRUDEntity(EntityORM) diff --git a/backend/src/app/core/data/crud/span_annotation.py b/backend/src/app/core/data/crud/span_annotation.py index 4847fca83..13840f622 100644 --- a/backend/src/app/core/data/crud/span_annotation.py +++ b/backend/src/app/core/data/crud/span_annotation.py @@ -5,11 +5,14 @@ from sqlalchemy.orm import Session from app.core.data.crud.annotation_document import crud_adoc +from app.core.data.crud.code import crud_code from app.core.data.crud.crud_base import CRUDBase +from app.core.data.crud.entity import crud_entity from app.core.data.crud.span_group import crud_span_group from app.core.data.crud.span_text import crud_span_text from app.core.data.dto.action import ActionType from app.core.data.dto.code import CodeRead +from app.core.data.dto.entity import EntityCreate, EntityMark from app.core.data.dto.span_annotation import ( SpanAnnotationCreate, SpanAnnotationCreateWithCodeId, @@ -35,6 +38,20 @@ def create( db=db, create_dto=SpanTextCreate(text=create_dto.span_text) ) + # create the entity + code = ( + db.query(CodeORM).filter(CodeORM.id == create_dto.current_code_id).first() + ) + project_id = code.project_id + crud_entity.create( + db=db, + create_dto=EntityCreate( + name=create_dto.span_text, + project_id=project_id, + span_text_ids=[span_text_orm.id], + ), + ) + # create the SpanAnnotation (and link the SpanText via FK) dto_obj_data = jsonable_encoder(create_dto.model_dump(exclude={"span_text"})) # noinspection PyArgumentList @@ -90,6 +107,23 @@ def create_multi( ], ) + # create the entities + code = crud_code.read(db=db, id=create_dtos[0].current_code_id) + project_id = code.project_id + try: + crud_entity.mark_multi( + db=db, + mark_dtos=[ + EntityMark( + project_id=project_id, + span_text_id=span_text_orm.id, + ) + for span_text_orm in span_texts_orm + ], + ) + except Exception as e: + raise Exception(e) + # create the SpanAnnotation (and link the SpanText via FK) dto_objs_data = [ jsonable_encoder(create_dto.model_dump(exclude={"span_text"})) diff --git a/backend/src/app/core/data/crud/span_text_entity_link.py b/backend/src/app/core/data/crud/span_text_entity_link.py new file mode 100644 index 000000000..c70bc8214 --- /dev/null +++ b/backend/src/app/core/data/crud/span_text_entity_link.py @@ -0,0 +1,55 @@ +from typing import Any, Dict, List, Optional + +import srsly +from fastapi.encoders import jsonable_encoder +from sqlalchemy.orm import Session + +from app.core.data.crud.crud_base import CRUDBase +from app.core.data.dto.span_text_entity_link import ( + SpanTextEntityLinkCreate, + SpanTextEntityLinkUpdate, +) +from app.core.data.orm.entity import EntityORM +from app.core.data.orm.span_text_entity_link import SpanTextEntityLinkORM +from config import conf + +# we need: +# create +# update +# delete +# read: +# by id, we can define the rest as it is needed + + +class CRUDSpanTextEntityLink( + CRUDBase[SpanTextEntityLinkORM, SpanTextEntityLinkCreate, SpanTextEntityLinkCreate] +): + def create( + self, db: Session, *, create_dto: SpanTextEntityLinkCreate + ) -> SpanTextEntityLinkORM: + dto_obj_data = jsonable_encoder(create_dto) + db_obj = self.model(**dto_obj_data) + db.add(db_obj) + db.commit() + return db_obj + + def update( + self, db: Session, *, id: int, update_dto: SpanTextEntityLinkUpdate + ) -> SpanTextEntityLinkUpdate: + return super().update(db, id=id, update_dto=update_dto) + + def read_by_span_text_id_and_project_id( + self, db: Session, *, span_text_id: int, project_id: int + ) -> SpanTextEntityLinkORM: + return ( + db.query(SpanTextEntityLinkORM) + .filter(SpanTextEntityLinkORM.linked_span_text_id == span_text_id) + .join(EntityORM, SpanTextEntityLinkORM.linked_entity_id == EntityORM.id) + .filter( + EntityORM.project_id == project_id, + ) + .first() + ) + + +crud_span_text_entity_link = CRUDSpanTextEntityLink(SpanTextEntityLinkORM) diff --git a/backend/src/app/core/data/dto/entity.py b/backend/src/app/core/data/dto/entity.py new file mode 100644 index 000000000..5377aee92 --- /dev/null +++ b/backend/src/app/core/data/dto/entity.py @@ -0,0 +1,68 @@ +from datetime import datetime +from typing import List + +from pydantic import BaseModel, ConfigDict, Field + +from app.core.data.dto.span_text import SpanTextRead + +from .dto_base import UpdateDTOBase + + +# Properties shared across all DTOs +class EntityBaseDTO(BaseModel): + pass + + +# Properties for creation +class EntityCreate(EntityBaseDTO): + name: str = Field(description="Name of the Entity") + project_id: int = Field(description="Project the Entity belongs to") + span_text_ids: List[int] = Field( + description="Span Text Ids which belong to this Entity" + ) + + +# Properties for updating +class EntityUpdate(EntityBaseDTO, UpdateDTOBase): + name: str = Field(description="Name of the Entity") + span_text_ids: List[int] = Field( + description="Span Text Ids which belong to this Entity" + ) + pass + + +class EntityMark(EntityBaseDTO): + span_text_id: int = Field(description="Span Text Id to mark") + project_id: int = Field(description="Project the Entity belongs to") + + +class EntityMerge(EntityBaseDTO): + name: str = Field(description="Name of the Entity") + project_id: int = Field(description="Id of the current Project") + entity_ids: List[int] = Field(description="List of Entity IDs to merge") + spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") + + +class EntityResolve(EntityBaseDTO): + project_id: int = Field(description="Id of the current Project") + entity_ids: List[int] = Field(description="List of Entity IDs to merge") + spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") + + +class EntityRelease(EntityBaseDTO): + name: str = Field(description="Name of the Entity") + entity_ids: List[int] = Field(description="List of Entity IDs to merge") + spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") + + +# Properties for reading (as in ORM) +class EntityRead(EntityBaseDTO): + id: int = Field(description="ID of the Entity") + name: str = Field(description="Name of the Entity") + project_id: int = Field(description="Project the Entity belongs to") + created: datetime = Field(description="Created timestamp of the Entity") + updated: datetime = Field(description="Updated timestamp of the Entity") + span_texts: List[SpanTextRead] = Field( + default=[], description="The SpanTexts belonging to this entity" + ) + model_config = ConfigDict(from_attributes=True) # TODO ask tim what this does diff --git a/backend/src/app/core/data/dto/span_text_entity_link.py b/backend/src/app/core/data/dto/span_text_entity_link.py new file mode 100644 index 000000000..13b7e6b61 --- /dev/null +++ b/backend/src/app/core/data/dto/span_text_entity_link.py @@ -0,0 +1,26 @@ +from typing import Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class SpanTextEntityLinkCreate(BaseModel): + linked_entity_id: Optional[int] = Field(description="ID of the linked Entity.") + linked_span_text_id: Optional[int] = Field( + description="ID of the linked span text." + ) + + +class SpanTextEntityLinkUpdate(BaseModel): + linked_entity_id: Optional[int] = Field(description="ID of the linked Entity.") + linked_span_text_id: Optional[int] = Field( + description="ID of the linked span text." + ) + + +class SpanTextEntityLinkRead(BaseModel): + id: int = Field(description="ID of the SpanTextEntityLink") + linked_entity_id: Optional[int] = Field(description="ID of the linked Entity.") + linked_span_text_id: Optional[int] = Field( + description="ID of the linked span text." + ) + model_config = ConfigDict(from_attributes=True) diff --git a/backend/src/app/core/data/orm/entity.py b/backend/src/app/core/data/orm/entity.py new file mode 100644 index 000000000..5608f91d5 --- /dev/null +++ b/backend/src/app/core/data/orm/entity.py @@ -0,0 +1,41 @@ +from datetime import datetime +from typing import TYPE_CHECKING, List, Optional + +from sqlalchemy import DateTime, ForeignKey, Integer, String, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.core.data.orm.orm_base import ORMBase + +if TYPE_CHECKING: + from app.core.data.orm.span_text import SpanTextORM + + +class EntityORM(ORMBase): + id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) + name: Mapped[str] = mapped_column(String, nullable=False, index=True) + created: Mapped[Optional[datetime]] = mapped_column( + DateTime, server_default=func.now(), index=True + ) + updated: Mapped[Optional[datetime]] = mapped_column( + DateTime, server_default=func.now(), onupdate=func.current_timestamp() + ) + + project_id: Mapped[int] = mapped_column( + Integer, + ForeignKey("project.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + span_texts: Mapped[List["SpanTextORM"]] = relationship( + "SpanTextORM", secondary="spantextentitylink" + ) + + +# __table_args__ = ( +# UniqueConstraint( +# "project_id", +# "name", +# name="UC_name_unique_per_project", +# ), +# ) diff --git a/backend/src/app/core/data/orm/span_text_entity_link.py b/backend/src/app/core/data/orm/span_text_entity_link.py new file mode 100644 index 000000000..ce3c0d017 --- /dev/null +++ b/backend/src/app/core/data/orm/span_text_entity_link.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import ForeignKey, Integer +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.data.orm.orm_base import ORMBase + +if TYPE_CHECKING: + pass + + +class SpanTextEntityLinkORM(ORMBase): + id = mapped_column(Integer, primary_key=True, index=True) + linked_entity_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("entity.id", ondelete="CASCADE"), index=True + ) + linked_span_text_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("spantext.id", ondelete="CASCADE"), index=True + ) diff --git a/backend/src/app/core/db/import_all_orms.py b/backend/src/app/core/db/import_all_orms.py index 6c297c54c..6445030b9 100644 --- a/backend/src/app/core/db/import_all_orms.py +++ b/backend/src/app/core/db/import_all_orms.py @@ -9,6 +9,7 @@ from app.core.data.orm.code import CodeORM from app.core.data.orm.concept_over_time_analysis import ConceptOverTimeAnalysisORM from app.core.data.orm.document_tag import DocumentTagORM +from app.core.data.orm.entity import EntityORM from app.core.data.orm.memo import MemoORM from app.core.data.orm.object_handle import ObjectHandleORM from app.core.data.orm.orm_base import ORMBase @@ -24,6 +25,7 @@ from app.core.data.orm.span_annotation import SpanAnnotationORM from app.core.data.orm.span_group import SpanGroupORM from app.core.data.orm.span_text import SpanTextORM +from app.core.data.orm.span_text_entity_link import SpanTextEntityLinkORM from app.core.data.orm.user import UserORM from app.core.data.orm.version import VersionORM from app.core.data.orm.whiteboard import WhiteboardORM diff --git a/backend/src/main.py b/backend/src/main.py index 2e1acd41b..28eeb537c 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -38,6 +38,7 @@ concept_over_time_analysis, crawler, document_tag, + entity, export, feedback, general, @@ -262,6 +263,7 @@ def invalid_error_handler(_, exc: InvalidError): app.include_router(span_group.router) app.include_router(bbox_annotation.router) app.include_router(code.router) +app.include_router(entity.router) app.include_router(memo.router) app.include_router(search.router) app.include_router(source_document_metadata.router) From 60d90db6a568e39f33662dc7317182ed60ce5a6b Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Tue, 25 Jun 2024 11:46:40 +0000 Subject: [PATCH 02/18] added frontend (entities) --- frontend/src/api/EntityHooks.ts | 70 +++++++ frontend/src/api/ProjectHooks.ts | 14 +- frontend/src/api/QueryKey.ts | 5 + .../src/components/entity/EntityTable.tsx | 189 ++++++++++++++++++ frontend/src/router/routes.tsx | 5 + frontend/src/views/analysis/Analysis.tsx | 7 + .../EntityDashboard/EntityDashboard.tsx | 102 ++++++++++ 7 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 frontend/src/api/EntityHooks.ts create mode 100644 frontend/src/components/entity/EntityTable.tsx create mode 100644 frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx diff --git a/frontend/src/api/EntityHooks.ts b/frontend/src/api/EntityHooks.ts new file mode 100644 index 000000000..41c8fb3d3 --- /dev/null +++ b/frontend/src/api/EntityHooks.ts @@ -0,0 +1,70 @@ +import { useMutation, useQuery } from "@tanstack/react-query"; +import queryClient from "../plugins/ReactQueryClient.ts"; +import { QueryKey } from "./QueryKey.ts"; +import { EntityRead } from "./openapi/models/EntityRead.ts"; +import { EntityService } from "./openapi/services/EntityService.ts"; + + +// enitity +const useGetEntity = (entityId: number | null | undefined) => + useQuery({ + queryKey: [QueryKey.ENTITY, entityId], + queryFn: () => EntityService.getById({ entityId: entityId! }), + enabled: !!entityId, + }); + +const useCreateEntity = () => + useMutation({ + mutationFn: EntityService.createNewEntity, + onSuccess: (newEntity, variables) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, variables.requestBody.project_id] }); + }, + }); + +const useUpdateEntity = () => + useMutation({ + mutationFn: EntityService.updateById, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data.id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id] }); + }, + }); + +const useDeleteEntity = () => + useMutation({ + mutationFn: EntityService.deleteById, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data.id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id] }); + }, + }); + +const useMerge = () => + useMutation({ + mutationFn: EntityService.mergeEntities, + onSuccess: (data) => { + queryClient.invalidateQueries({queryKey: [QueryKey.ENTITY, data.project_id]}); + queryClient.invalidateQueries({queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id]}) + } + }) + + const useResolve = () => + useMutation({ + mutationFn: EntityService.resolveEntities, + onSuccess: (data) => { + queryClient.invalidateQueries({queryKey: [QueryKey.ENTITY, data[0].project_id]}); + queryClient.invalidateQueries({queryKey: [QueryKey.PROJECT_ENTITIES, data[0].project_id]}) + } + }) + + +const EntityHooks = { + useGetEntity, + useCreateEntity, + useUpdateEntity, + useDeleteEntity, + useMerge, + useResolve +}; + +export default EntityHooks; diff --git a/frontend/src/api/ProjectHooks.ts b/frontend/src/api/ProjectHooks.ts index 22723a896..8b3fd7e04 100644 --- a/frontend/src/api/ProjectHooks.ts +++ b/frontend/src/api/ProjectHooks.ts @@ -7,6 +7,7 @@ import { ActionQueryParameters } from "./openapi/models/ActionQueryParameters.ts import { ActionRead } from "./openapi/models/ActionRead.ts"; import { CodeRead } from "./openapi/models/CodeRead.ts"; import { DocumentTagRead } from "./openapi/models/DocumentTagRead.ts"; +import { EntityRead } from "./openapi/models/EntityRead.ts"; import { MemoRead } from "./openapi/models/MemoRead.ts"; import { PreprocessingJobRead } from "./openapi/models/PreprocessingJobRead.ts"; import { ProjectCreate } from "./openapi/models/ProjectCreate.ts"; @@ -146,7 +147,16 @@ const useGetAllCodes = (projectId: number, returnAll: boolean = false) => { select: returnAll ? undefined : selectEnabledCodes, }); }; - +// entities +const useGetAllEntities = (projectId: number) => { + return useQuery({ + queryKey: [QueryKey.PROJECT_ENTITIES, projectId], + queryFn: () => + ProjectService.getProjectEntities({ + projId: projectId, + }), + }); +}; // memo const useGetMemo = (projectId: number | null | undefined, userId: number | null | undefined) => useQuery({ @@ -230,6 +240,8 @@ const ProjectHooks = { useRemoveUser, // codes useGetAllCodes, + // entities, + useGetAllEntities, // memo useGetMemo, useGetAllUserMemos, diff --git a/frontend/src/api/QueryKey.ts b/frontend/src/api/QueryKey.ts index b1c8c6697..572842021 100644 --- a/frontend/src/api/QueryKey.ts +++ b/frontend/src/api/QueryKey.ts @@ -11,6 +11,8 @@ export const QueryKey = { PROJECT_SDOCS_INFINITE: "projectDocumentsInfinite", // all codes of a project (by project id) PROJECT_CODES: "projectCodes", + // all entities of a project (by project id) + PROJECT_ENTITIES: "projectEntities", // all tags of a project (by project id) PROJECT_TAGS: "projectTags", // all crawler jobs of a project (by project id) @@ -108,6 +110,9 @@ export const QueryKey = { // a single code (by code id) CODE: "code", + // a single entity (by entity id) + ENTITY: "entity", + // a single tag (by tag id) TAG: "tag", diff --git a/frontend/src/components/entity/EntityTable.tsx b/frontend/src/components/entity/EntityTable.tsx new file mode 100644 index 000000000..91a35075c --- /dev/null +++ b/frontend/src/components/entity/EntityTable.tsx @@ -0,0 +1,189 @@ +import { + MRT_ColumnDef, + MRT_RowSelectionState, + MRT_TableInstance, + MRT_TableOptions, + MaterialReactTable, + useMaterialReactTable, +} from "material-react-table"; +import { useMemo } from "react"; +import ProjectHooks from "../../api/ProjectHooks.ts"; +import { EntityRead } from "../../api/openapi/models/EntityRead.ts"; +import { SpanTextRead } from "../../api/openapi/models/SpanTextRead.ts"; + +interface EnitityTableRow{ + id: string, + original: EntityRead, + subRows: SpanTextTableRow[]; + editable: boolean; +} + +interface SpanTextTableRow{ + id: string, + original: SpanTextRead, + subRows: SpanTextTableRow[]; + editable: boolean; +} + +const columns: MRT_ColumnDef[] = [ + { + accessorKey: "original.id", + header: "ID", + enableEditing: false, + }, + { + accessorKey: 'original.name', + header: 'Name', + enableEditing: true, + }, +]; + +export interface EntityTableActionProps { + table: MRT_TableInstance; + selectedEntities: EntityRead[]; + selectedSpanTexts: SpanTextRead[]; +} + +export interface EntityTableProps { + projectId: number; + // selection + enableMultiRowSelection?: boolean; + rowSelectionModel: MRT_RowSelectionState; + onRowSelectionChange: MRT_TableOptions["onRowSelectionChange"]; + // toolbar + renderToolbarInternalActions?: (props: EntityTableActionProps) => React.ReactNode; + renderTopToolbarCustomActions?: (props: EntityTableActionProps) => React.ReactNode; + renderBottomToolbarCustomActions?: (props: EntityTableActionProps) => React.ReactNode; + // editing + onSaveEditRow: MRT_TableOptions['onEditingRowSave']; +} + +function EntityTable({ + projectId, + enableMultiRowSelection = true, + rowSelectionModel, + onRowSelectionChange, + renderToolbarInternalActions, + renderTopToolbarCustomActions, + renderBottomToolbarCustomActions, + onSaveEditRow, +}: EntityTableProps) { + // global server state + const projectEntities = ProjectHooks.useGetAllEntities(projectId); + + // computed + const { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap } = useMemo(() => { + if (!projectEntities.data) return { projectEntitiesMap: {} as Record, projectEntitiesRows: [], projectSpanTextMap: {} as Record}; + + const projectEntitiesMap = projectEntities.data.reduce( + (entity_map, projectEntity) => { + const id = `E-${projectEntity.id}`; + entity_map[id] = projectEntity; + return entity_map; + }, + {} as Record, + ); + const projectEntitiesRows = projectEntities.data.map(entity => { + const subRows = entity.span_texts?.map(span => ({ + id: `S-${span.id}`, + original: {...span, name: span.text}, + subRows: [], + editable: false, + })) || []; + const original = entity; + const id = `E-${entity.id}`; + const editable = true; + return { id, original, subRows, editable }; + }); + + const projectSpanTextMap = projectEntities.data.reduce((acc, entity) => { + if (Array.isArray(entity.span_texts)) { + entity.span_texts.forEach(span => { + acc[`S-${span.id}`] = span; + }); + } + return acc; + }, {} as Record) + + return { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap}; + }, [projectEntities.data]); + + // table + const table = useMaterialReactTable({ + data: projectEntitiesRows, + columns: columns, + getRowId: (row) => `${row.id}`, + enableEditing: (row) => {return row.original.editable}, + editDisplayMode: 'row', + onEditingRowSave: onSaveEditRow, + // style + muiTablePaperProps: { + elevation: 0, + style: { height: "100%", display: "flex", flexDirection: "column" }, + }, + muiTableContainerProps: { + style: { flexGrow: 1 }, + }, + // state + state: { + rowSelection: rowSelectionModel, + isLoading: projectEntities.isLoading, + showAlertBanner: projectEntities.isError, + showProgressBars: projectEntities.isFetching, + }, + // handle error + muiToolbarAlertBannerProps: projectEntities.isError + ? { + color: "error", + children: projectEntities.error.message, + } + : undefined, + // virtualization (scrolling instead of pagination) + enablePagination: false, + enableRowVirtualization: true, + // selection + enableRowSelection: true, + enableMultiRowSelection: enableMultiRowSelection, + onRowSelectionChange, + // toolbar + enableBottomToolbar: true, + renderTopToolbarCustomActions: renderTopToolbarCustomActions + ? (props) => + renderTopToolbarCustomActions({ + table: props.table, + selectedEntities: Object.keys(rowSelectionModel).filter(id => id.startsWith('E-')).map((entityId) => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(rowSelectionModel).filter(id => id.startsWith('S-')).map((spanTextId) => projectSpanTextMap[spanTextId]), + }) + : undefined, + renderToolbarInternalActions: renderToolbarInternalActions + ? (props) => + renderToolbarInternalActions({ + table: props.table, + selectedEntities: Object.keys(rowSelectionModel).filter(id => id.startsWith('E-')).map((entityId) => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(rowSelectionModel).filter(id => id.startsWith('S-')).map((spanTextId) => projectSpanTextMap[spanTextId]), + }) + : undefined, + renderBottomToolbarCustomActions: renderBottomToolbarCustomActions + ? (props) => + renderBottomToolbarCustomActions({ + table: props.table, + selectedEntities: Object.keys(rowSelectionModel).filter(id => id.startsWith('E-')).map((entityId) => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(rowSelectionModel).filter(id => id.startsWith('S-')).map((spanTextId) => projectSpanTextMap[spanTextId]), + }) + : undefined, + // hide columns per default + initialState: { + columnVisibility: { + id: false, + }, + }, + // tree structure + enableExpanding: true, + getSubRows: (originalRow) => originalRow.subRows, + filterFromLeafRows: true, //search for child rows and preserve parent rows + enableSubRowSelection: false, + }); + + return ; +} +export default EntityTable; diff --git a/frontend/src/router/routes.tsx b/frontend/src/router/routes.tsx index b687e113c..8533c781b 100644 --- a/frontend/src/router/routes.tsx +++ b/frontend/src/router/routes.tsx @@ -13,6 +13,7 @@ import CodeFrequencyAnalysis from "../views/analysis/CodeFrequency/CodeFrequency import CodeGraph from "../views/analysis/CodeGraph/CodeGraph.tsx"; import CotaDashboard from "../views/analysis/ConceptsOverTime/CotaDashboard.tsx"; import CotaView from "../views/analysis/ConceptsOverTime/CotaView.tsx"; +import EntityDashboard from "../views/analysis/EntityDashboard/EntityDashboard.tsx"; import TableDashboard from "../views/analysis/Table/TableDashboard.tsx"; import TableView from "../views/analysis/Table/TableView.tsx"; import TimelineAnalysis from "../views/analysis/TimelineAnalysis/TimelineAnalysis.tsx"; @@ -151,6 +152,10 @@ const router = createBrowserRouter([ path: "/project/:projectId/analysis/annotated-segments", element: , }, + { + path: "/project/:projectId/analysis/entity-dashboard", + element: , + }, { path: "/project/:projectId/analysis/word-frequency", element: , diff --git a/frontend/src/views/analysis/Analysis.tsx b/frontend/src/views/analysis/Analysis.tsx index fd3455952..34b3b6bff 100644 --- a/frontend/src/views/analysis/Analysis.tsx +++ b/frontend/src/views/analysis/Analysis.tsx @@ -58,6 +58,13 @@ function Analysis() { color={"#77dd77"} /> + + diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx new file mode 100644 index 000000000..b2a23e54c --- /dev/null +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -0,0 +1,102 @@ +import { Box, Button, Grid, Portal, Stack, Typography } from "@mui/material"; +import { MRT_RowSelectionState } from "material-react-table"; +import { useContext, useState } from "react"; +import { useParams } from "react-router-dom"; +import EntityHooks from "../../../api/EntityHooks.ts"; +import { EntityRead } from "../../../api/openapi/models/EntityRead.ts"; +import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; +import EntityTable from "../../../components/entity/EntityTable.tsx"; +import { AppBarContext } from "../../../layouts/TwoBarLayout.tsx"; +import { useAppSelector } from "../../../plugins/ReduxHooks.ts"; + + +function EntityDashboard() { + const appBarContainerRef = useContext(AppBarContext); + + // global client state (react router) + const projectId = parseInt(useParams<{ projectId: string }>().projectId!); + + // global client state (redux) + const isSplitView = useAppSelector((state) => state.annotatedSegments.isSplitView); + const [rowSelectionModel, setRowSelectionModel] = useState({}); + const entityMerge = EntityHooks.useMerge(); + const entityResolve = EntityHooks.useResolve(); + const entityUpdate = EntityHooks.useUpdateEntity(); + + + + + function handleMerg(selectedEntities: EntityRead[], selectedSpanTexts: SpanTextRead[]): void { + const name = "merge" + selectedEntities[0]?.name + selectedSpanTexts[0]?.text; + const requestBody = { + requestBody: { + name: name, + project_id: projectId, + entity_ids: selectedEntities.map(entity => entity.id), + spantext_ids: selectedSpanTexts.map(spantext => spantext.id) + } + }; + entityMerge.mutate(requestBody); + setRowSelectionModel({}); + } + + function handleRelease(selectedEntities: EntityRead[], selectedSpanTexts: SpanTextRead[]): void { + console.log(rowSelectionModel); + console.log(selectedEntities); + console.log(selectedSpanTexts); + const requestBody = { + requestBody: { + project_id: projectId, + entity_ids: selectedEntities.map(entity => entity.id), + spantext_ids: selectedSpanTexts.map(spantext => spantext.id) + } + }; + entityResolve.mutate(requestBody); + setRowSelectionModel({}); + } + + + + function handleUpdate(props: any): void | Promise { + // TODO fix naming + const requestBody = + { + entityId: props.values["original.id"], + requestBody: { + name: props.values["original.name"], + span_text_ids: props.row.original.original.span_texts.map(span_text => span_text.id) + } + }; + console.log(requestBody); + entityUpdate.mutate(requestBody); + props.table.setEditingRow(null); + } + + return ( + + + + Entity Dashboard + + + + + + ( + + + + + + )} onSaveEditRow={handleUpdate} /> + + + + ); +} + +export default EntityDashboard; From e6ab5f1f0ed0611de18388ea8c380734d7cbf9a8 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Tue, 25 Jun 2024 23:30:30 +0000 Subject: [PATCH 03/18] update backend --- backend/src/api/endpoints/entity.py | 107 +++----- backend/src/api/endpoints/project.py | 16 -- backend/src/app/core/data/crud/entity.py | 259 +++--------------- .../src/app/core/data/crud/span_annotation.py | 13 +- .../core/data/crud/span_text_entity_link.py | 87 +++++- backend/src/app/core/data/dto/entity.py | 15 +- 6 files changed, 170 insertions(+), 327 deletions(-) diff --git a/backend/src/api/endpoints/entity.py b/backend/src/api/endpoints/entity.py index 6591ddd29..5fbe7943b 100644 --- a/backend/src/api/endpoints/entity.py +++ b/backend/src/api/endpoints/entity.py @@ -1,63 +1,27 @@ +from itertools import chain from typing import List from fastapi import APIRouter, Depends from sqlalchemy.orm import Session from api.dependencies import get_current_user, get_db_session -from api.util import get_object_memo_for_user, get_object_memos -from api.validation import Validate from app.core.authorization.authz_user import AuthzUser from app.core.data.crud import Crud from app.core.data.crud.entity import crud_entity -from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link +from app.core.data.crud.span_text import crud_span_text from app.core.data.dto.entity import ( EntityCreate, EntityMerge, EntityRead, - EntityResolve, + EntityRelease, EntityUpdate, ) -from app.core.data.dto.span_text_entity_link import SpanTextEntityLinkCreate router = APIRouter( prefix="/entity", dependencies=[Depends(get_current_user)], tags=["entity"] ) -@router.put( - "", - response_model=EntityRead, - summary="Creates a new Entity and returns it with the generated ID.", -) -def create_new_entity( - *, - db: Session = Depends(get_db_session), - entity: EntityCreate, - authz_user: AuthzUser = Depends(), -) -> EntityRead: - authz_user.assert_in_project(entity.project_id) - - db_obj = crud_entity.create(db=db, create_dto=entity) - - return EntityRead.model_validate(db_obj) - - -@router.get( - "/{entity_id}", - response_model=EntityRead, - summary="Returns the Entity with the given ID.", -) -def get_by_id( - *, - db: Session = Depends(get_db_session), - entity_id: int, - authz_user: AuthzUser = Depends(), -) -> EntityRead: - authz_user.assert_in_same_project_as(Crud.ENTITY, entity_id) - db_obj = crud_entity.read(db=db, id=entity_id) - return EntityRead.model_validate(db_obj) - - @router.patch( "/{entity_id}", response_model=EntityRead, @@ -80,7 +44,7 @@ def update_by_id( @router.put( "/merge", response_model=EntityRead, - summary="Merges entities with given IDs.", + summary="Merges entities and/or span texts with given IDs.", ) def merge_entities( *, @@ -88,42 +52,55 @@ def merge_entities( entity_merge: EntityMerge, authz_user: AuthzUser = Depends(), ) -> EntityRead: - print("merge_entities") authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_merge.entity_ids) - db_obj = crud_entity.merge(db=db, merge_dto=entity_merge) + all_span_texts = ( + list( + chain.from_iterable( + [st.id for st in crud_entity.read(db=db, id=id).span_texts] + for id in entity_merge.entity_ids + ) + ) + + entity_merge.spantext_ids + ) + new_entity = EntityCreate( + name=entity_merge.name, + project_id=entity_merge.project_id, + span_text_ids=all_span_texts, + ) + db_obj = crud_entity.create(db=db, create_dto=new_entity, force=True) return EntityRead.model_validate(db_obj) # add resolve endpoint @router.put( - "/resolve", + "/release", response_model=List[EntityRead], - summary="Resolve entities with given IDs.", + summary="Releases entities and/or span texts with given IDs.", ) -def resolve_entities( +def release_entities( *, db: Session = Depends(get_db_session), - entity_resolve: EntityResolve, + entity_resolve: EntityRelease, authz_user: AuthzUser = Depends(), ) -> EntityRead: - print("resolve_entities") authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_resolve.entity_ids) - db_objs = crud_entity.resolve(db=db, resolve_dto=entity_resolve) + all_span_texts = ( + list( + chain.from_iterable( + [st.id for st in crud_entity.read(db=db, id=id).span_texts] + for id in entity_resolve.entity_ids + ) + ) + + entity_resolve.spantext_ids + ) + new_entities = [] + for span_text_id in all_span_texts: + span_text = crud_span_text.read(db=db, id=span_text_id) + new_entity = EntityCreate( + name=span_text.text, + project_id=entity_resolve.project_id, + span_text_ids=[span_text_id], + ) + new_entities.append(new_entity) + db_objs = crud_entity.create_multi(db=db, create_dtos=new_entities, force=True) return [EntityRead.model_validate(db_obj) for db_obj in db_objs] - - -@router.delete( - "/{entity_id}", - response_model=EntityRead, - summary="Deletes the Entity with the given ID.", -) -def delete_by_id( - *, - db: Session = Depends(get_db_session), - entity_id: int, - authz_user: AuthzUser = Depends(), -) -> EntityRead: - authz_user.assert_in_same_project_as(Crud.ENTITY, entity_id) - - db_obj = crud_entity.remove(db=db, id=entity_id) - return EntityRead.model_validate(db_obj) diff --git a/backend/src/api/endpoints/project.py b/backend/src/api/endpoints/project.py index b44011671..7619acb6a 100644 --- a/backend/src/api/endpoints/project.py +++ b/backend/src/api/endpoints/project.py @@ -551,19 +551,3 @@ def get_project_entities( result = [EntityRead.model_validate(entity) for entity in result] result.sort(key=lambda c: c.id) return result - - -@router.delete( - "/{proj_id}/entity", - response_model=List[int], - summary="Removes all Entities of the Project with the given ID if it exists", -) -def delete_project_entity( - *, - proj_id: int, - db: Session = Depends(get_db_session), - authz_user: AuthzUser = Depends(), -) -> List[int]: - authz_user.assert_in_project(proj_id) - - return crud_entity.remove_by_project(db=db, proj_id=proj_id) diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py index 06321f24c..0946ff36e 100644 --- a/backend/src/app/core/data/crud/entity.py +++ b/backend/src/app/core/data/crud/entity.py @@ -1,254 +1,75 @@ -from typing import Any, Dict, List, Optional +from typing import List, Optional -import srsly from fastapi.encoders import jsonable_encoder +from sqlalchemy import select from sqlalchemy.orm import Session from app.core.data.crud.crud_base import CRUDBase -from app.core.data.crud.span_text import crud_span_text from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link -from app.core.data.dto.action import ActionType from app.core.data.dto.entity import ( EntityCreate, - EntityMark, - EntityMerge, - EntityRead, - EntityResolve, EntityUpdate, ) from app.core.data.dto.span_text_entity_link import ( SpanTextEntityLinkCreate, - SpanTextEntityLinkUpdate, ) from app.core.data.orm.entity import EntityORM -from config import conf +from app.core.data.orm.span_text_entity_link import SpanTextEntityLinkORM class CRUDEntity(CRUDBase[EntityORM, EntityCreate, EntityUpdate]): - def create(self, db: Session, *, create_dto: EntityCreate) -> EntityORM: - dto_obj_data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) - db_obj = self.model(**dto_obj_data) - db.add(db_obj) - db.flush() - for span_text_id in create_dto.span_text_ids: - possible_link = ( - crud_span_text_entity_link.read_by_span_text_id_and_project_id( - db=db, span_text_id=span_text_id, project_id=create_dto.project_id - ) - ) - if possible_link is not None: - link = crud_span_text_entity_link.update( - db=db, - id=possible_link.id, - update_dto=SpanTextEntityLinkUpdate( - linked_entity_id=db_obj.id, - linked_span_text_id=possible_link.linked_span_text_id, - ), - ) - else: - link = crud_span_text_entity_link.create( - db=db, - create_dto=SpanTextEntityLinkCreate( - linked_entity_id=db_obj.id, linked_span_text_id=span_text_id - ), - ) - db.add(link) - db.commit() - db.refresh(db_obj) - return db_obj + def create( + self, db: Session, *, create_dto: EntityCreate, force: bool = True + ) -> EntityORM: + result = self.create_multi(db=db, create_dtos=[create_dto], force=force) + return result[0] if len(result) > 0 else None def create_multi( - self, db: Session, *, create_dtos: List[EntityCreate] + self, db: Session, *, create_dtos: List[EntityCreate], force: bool = True ) -> List[EntityORM]: - db_objs = [] - for create_dto in create_dtos: - dto_obj_data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) - db_obj = self.model(**dto_obj_data) - db.add(db_obj) - db_objs.append(db_obj) + if len(create_dtos) == 0: + return [] + dto_objs_data = [ + jsonable_encoder(dto, exclude={"span_text_ids"}) for dto in create_dtos + ] + db_objs = [self.model(**data) for data in dto_objs_data] + db.add_all(db_objs) db.flush() + db.commit() + links = [] for db_obj, create_dto in zip(db_objs, create_dtos): for span_text_id in create_dto.span_text_ids: - possible_link = ( - crud_span_text_entity_link.read_by_span_text_id_and_project_id( - db=db, - span_text_id=span_text_id, - project_id=create_dto.project_id, + links.append( + SpanTextEntityLinkCreate( + linked_entity_id=db_obj.id, linked_span_text_id=span_text_id ) ) - if possible_link is not None: - link = crud_span_text_entity_link.update( - db=db, - id=possible_link.id, - update_dto=SpanTextEntityLinkUpdate( - linked_entity_id=db_obj.id, - linked_span_text_id=possible_link.linked_span_text_id, - ), - ) - else: - link = crud_span_text_entity_link.create( - db=db, - create_dto=SpanTextEntityLinkCreate( - linked_entity_id=db_obj.id, linked_span_text_id=span_text_id - ), - ) - db.add(link) - + crud_span_text_entity_link.create_multi(db=db, create_dtos=links, force=force) db.commit() + self.remove_all_unused_entites(db=db) return db_objs - def mark(self, db: Session, *, mark_dto: EntityMark) -> EntityORM | None: - span_text_link = crud_span_text_entity_link.read_by_span_text_id_and_project_id( - db=db, span_text_id=mark_dto.span_text_id, project_id=mark_dto.project_id - ) - if span_text_link is None: - span_text = crud_span_text.read(db=db, id=mark_dto.span_text_id) - entity_orm = self.create( - db=db, - create_dto=EntityCreate( - name=span_text.text, - project_id=mark_dto.project_id, - span_text_ids=[mark_dto.span_text_id], - ), - ) - return entity_orm - return self.read(db=db, id=span_text_link.linked_entity_id) - - def mark_multi( - self, db: Session, *, mark_dtos: List[EntityMark] - ) -> List[EntityORM]: - to_update = {} - for entity_mark in mark_dtos: - span_text_link = ( - crud_span_text_entity_link.read_by_span_text_id_and_project_id( - db=db, - span_text_id=entity_mark.span_text_id, - project_id=entity_mark.project_id, - ) - ) - if span_text_link is None and entity_mark.span_text_id not in to_update: - span_text = crud_span_text.read(db=db, id=entity_mark.span_text_id) - to_update[entity_mark.span_text_id] = EntityCreate( - name=span_text.text, - project_id=entity_mark.project_id, - span_text_ids=[entity_mark.span_text_id], - ) - return self.create_multi(db=db, create_dtos=to_update.values()) - - def merge(self, db: Session, *, merge_dto: EntityMerge) -> EntityORM | None: - project_id = merge_dto.project_id - entities_to_delete = merge_dto.entity_ids[:] - span_text_ids = merge_dto.spantext_ids[:] - - for span_text_id in merge_dto.spantext_ids: - entity = self.read_by_span_text_id_and_project_id( - db=db, span_text_id=span_text_id, project_id=project_id - ) - assert ( - entity is not None - ), "SpanText given does not belong to an entity withing the current Project\nproject_id: {project_id}\nspan_text: {span_text}" - if len(entity.span_texts) <= 1: - entities_to_delete.append(entity.id) - - for entity_id in merge_dto.entity_ids: - entity = self.read(db=db, id=entity_id) - if entity is not None: - assert ( - entity.project_id == project_id - ), "Entity given is not in current Project\nproject_id: {project_id}\nentity: {entity}" - for span_text in entity.span_texts: - span_text_ids.append(span_text.id) - new_entity = self.create( - db=db, - create_dto=EntityCreate( - name=merge_dto.name, - project_id=project_id, - span_text_ids=span_text_ids, - ), - ) - self.remove_multi(db=db, ids=entities_to_delete) - return new_entity - - def resolve(self, db: Session, *, resolve_dto: EntityResolve) -> EntityORM | None: - entities_to_create = [] - entities_to_remove = [] - project_id = resolve_dto.project_id - for span_text_id in resolve_dto.spantext_ids: - entity = self.read_by_span_text_id_and_project_id( - db=db, span_text_id=span_text_id, project_id=project_id - ) - span_text = crud_span_text.read(db=db, id=span_text_id) - assert ( - entity is not None - ), "SpanText given does not belong to an entity withing the current Project\nproject_id: {project_id}\nspan_text: {span_text}" - entities_to_create.append( - EntityCreate( - name=span_text.text, - project_id=project_id, - span_text_ids=[span_text_id], - ) - ) - if len(entity.span_texts) <= 1: - entities_to_remove.append(entity.id) - for entity_id in resolve_dto.entity_ids: - entity = self.read(db=db, id=entity_id) - assert ( - entity.project_id == project_id - ), "Entity given is not in current Project\nproject_id: {project_id}\nentity: {entity}" - entities_to_remove.append(entity_id) - for span_text in entity.span_texts: - entities_to_create.append( - EntityCreate( - name=span_text.text, - project_id=project_id, - span_text_ids=[span_text.id], - ) - ) - self.remove_multi(db=db, ids=entities_to_remove) - new_entities = self.create_multi(db=db, create_dtos=entities_to_create) - - return new_entities - - def update( - self, db: Session, *, id: int, update_dto: EntityUpdate - ) -> EntityORM | None: - return super().update(db, id=id, update_dto=update_dto) - - def read_by_name_and_project( - self, db: Session, entity_name: str, proj_id: int - ) -> Optional[EntityORM]: - return ( - db.query(self.model) - .filter(self.model.name == entity_name, self.model.project_id == proj_id) - .first() - ) - - def read_by_span_text_id_and_project_id( - self, db: Session, *, span_text_id: int, project_id: int - ) -> EntityORM: - span_text_link = crud_span_text_entity_link.read_by_span_text_id_and_project_id( - db=db, span_text_id=span_text_id, project_id=project_id - ) - return self.read(db=db, id=span_text_link.linked_entity_id) - def read_by_project(self, db: Session, proj_id: int) -> List[EntityORM]: return db.query(self.model).filter(self.model.project_id == proj_id).all() - def read_by_id(self, db: Session, entity_id: int) -> Optional[EntityORM]: - return db.query(self.model).filter(self.model.id == entity_id).first() - - def remove_by_project(self, db: Session, *, proj_id: int) -> List[EntityORM]: - query = db.query(self.model).filter(self.model.project_id == proj_id) - removed_orms = query.all() - ids = [removed_orm.id for removed_orm in removed_orms] - self.remove_multi(db=db, ids=ids) - return ids - - def remove_multi(self, db: Session, *, ids: List[int]) -> bool: - for id in ids: - self.remove(db=db, id=id) - return True + def remove_multi(self, db: Session, *, ids: List[int]) -> List[EntityORM]: + removed = db.query(EntityORM).filter(EntityORM.id.in_(ids)).all() + db.query(EntityORM).filter(EntityORM.id.in_(ids)).delete( + synchronize_session=False + ) + db.commit() + return removed + + def remove_all_unused_entites(self, db: Session) -> List[EntityORM]: + subquery = select(SpanTextEntityLinkORM.linked_entity_id).distinct().subquery() + query = ( + db.query(EntityORM) + .outerjoin(subquery, EntityORM.id == subquery.c.linked_entity_id) + .filter(subquery.c.linked_entity_id.is_(None)) + ) + to_remove = query.all() + return self.remove_multi(db=db, ids=[e.id for e in to_remove]) crud_entity = CRUDEntity(EntityORM) diff --git a/backend/src/app/core/data/crud/span_annotation.py b/backend/src/app/core/data/crud/span_annotation.py index 13840f622..83343ee00 100644 --- a/backend/src/app/core/data/crud/span_annotation.py +++ b/backend/src/app/core/data/crud/span_annotation.py @@ -12,7 +12,7 @@ from app.core.data.crud.span_text import crud_span_text from app.core.data.dto.action import ActionType from app.core.data.dto.code import CodeRead -from app.core.data.dto.entity import EntityCreate, EntityMark +from app.core.data.dto.entity import EntityCreate from app.core.data.dto.span_annotation import ( SpanAnnotationCreate, SpanAnnotationCreateWithCodeId, @@ -111,14 +111,15 @@ def create_multi( code = crud_code.read(db=db, id=create_dtos[0].current_code_id) project_id = code.project_id try: - crud_entity.mark_multi( + crud_entity.create_multi( db=db, - mark_dtos=[ - EntityMark( + create_dtos=[ + EntityCreate( project_id=project_id, - span_text_id=span_text_orm.id, + name=dto.span_text, + span_text_ids=[id[0]], ) - for span_text_orm in span_texts_orm + for id, dto in zip(span_texts_orm, create_dtos) ], ) except Exception as e: diff --git a/backend/src/app/core/data/crud/span_text_entity_link.py b/backend/src/app/core/data/crud/span_text_entity_link.py index c70bc8214..687d3f8e3 100644 --- a/backend/src/app/core/data/crud/span_text_entity_link.py +++ b/backend/src/app/core/data/crud/span_text_entity_link.py @@ -1,6 +1,5 @@ -from typing import Any, Dict, List, Optional +from typing import List -import srsly from fastapi.encoders import jsonable_encoder from sqlalchemy.orm import Session @@ -11,7 +10,6 @@ ) from app.core.data.orm.entity import EntityORM from app.core.data.orm.span_text_entity_link import SpanTextEntityLinkORM -from config import conf # we need: # create @@ -33,23 +31,94 @@ def create( db.commit() return db_obj + def create_multi( + self, db: Session, *, create_dtos: List[SpanTextEntityLinkCreate], force: bool + ) -> List[SpanTextEntityLinkORM]: + # One assumption is that all entities have the same project_id + if len(create_dtos) == 0: + return [] + project_id = ( + db.query(EntityORM) + .filter(EntityORM.id == create_dtos[0].linked_entity_id) + .first() + .project_id + ) + all_ids = [link.linked_span_text_id for link in create_dtos] + existing_links = self.read_multi_span_text_and_project_id( + db=db, span_text_ids=all_ids, project_id=project_id + ) + existing_ids = [d.linked_span_text_id for d in existing_links] + to_create = { + dto.linked_span_text_id: dto + for dto in create_dtos + if dto.linked_span_text_id not in existing_ids + } + to_create = to_create.values() + + if len(to_create) > 0: + db_objs = [self.model(**jsonable_encoder(dto)) for dto in to_create] + db.bulk_save_objects(db_objs) + db.commit() + if len(existing_links) > 0: + if force: + existing_links_map = { + link.linked_span_text_id: link for link in existing_links + } + for create_dto in create_dtos: + if create_dto.linked_span_text_id in existing_links_map: + existing_links_map[ + create_dto.linked_span_text_id + ].linked_entity_id = create_dto.linked_entity_id + ids = [dto.id for dto in existing_links] + update_dtos = [ + SpanTextEntityLinkUpdate( + linked_entity_id=dto.linked_entity_id, + linked_span_text_id=dto.linked_span_text_id, + ) + for dto in existing_links + ] + self.update_multi(db, ids=ids, update_dtos=update_dtos) + def update( self, db: Session, *, id: int, update_dto: SpanTextEntityLinkUpdate - ) -> SpanTextEntityLinkUpdate: + ) -> SpanTextEntityLinkORM: return super().update(db, id=id, update_dto=update_dto) - def read_by_span_text_id_and_project_id( - self, db: Session, *, span_text_id: int, project_id: int + def update_multi( + self, + db: Session, + *, + ids: List[int], + update_dtos: List[SpanTextEntityLinkUpdate], ) -> SpanTextEntityLinkORM: - return ( + if len(ids) != len(update_dtos): + raise ValueError("The number of IDs must match the number of update DTOs") + + update_mappings = [] + for id, dto in zip(ids, update_dtos): + dto_data = jsonable_encoder(dto) + dto_data["id"] = id + update_mappings.append(dto_data) + + db.bulk_update_mappings(self.model, update_mappings) + db.commit() + + updated_records = db.query(self.model).filter(self.model.id.in_(ids)).all() + return updated_records + + def read_multi_span_text_and_project_id( + self, db: Session, *, span_text_ids: List[int], project_id: int + ) -> List[SpanTextEntityLinkORM]: + query = ( db.query(SpanTextEntityLinkORM) - .filter(SpanTextEntityLinkORM.linked_span_text_id == span_text_id) .join(EntityORM, SpanTextEntityLinkORM.linked_entity_id == EntityORM.id) .filter( + SpanTextEntityLinkORM.linked_span_text_id.in_(span_text_ids), EntityORM.project_id == project_id, ) - .first() + .distinct() ) + return query.all() crud_span_text_entity_link = CRUDSpanTextEntityLink(SpanTextEntityLinkORM) diff --git a/backend/src/app/core/data/dto/entity.py b/backend/src/app/core/data/dto/entity.py index 5377aee92..fa9d2946b 100644 --- a/backend/src/app/core/data/dto/entity.py +++ b/backend/src/app/core/data/dto/entity.py @@ -31,11 +31,7 @@ class EntityUpdate(EntityBaseDTO, UpdateDTOBase): pass -class EntityMark(EntityBaseDTO): - span_text_id: int = Field(description="Span Text Id to mark") - project_id: int = Field(description="Project the Entity belongs to") - - +# Properties for merging entities/span texts class EntityMerge(EntityBaseDTO): name: str = Field(description="Name of the Entity") project_id: int = Field(description="Id of the current Project") @@ -43,14 +39,9 @@ class EntityMerge(EntityBaseDTO): spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") -class EntityResolve(EntityBaseDTO): - project_id: int = Field(description="Id of the current Project") - entity_ids: List[int] = Field(description="List of Entity IDs to merge") - spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") - - +# Properties for releasing entities/span texts class EntityRelease(EntityBaseDTO): - name: str = Field(description="Name of the Entity") + project_id: int = Field(description="Id of the current Project") entity_ids: List[int] = Field(description="List of Entity IDs to merge") spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") From 31b49c8c1cd3c9d4e7e3ccc1d2fa2138f4c97150 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Tue, 25 Jun 2024 23:38:23 +0000 Subject: [PATCH 04/18] update frontend --- .../src/components/entity/EntityTable.tsx | 41 ++++++++---- .../EntityDashboard/EntityDashboard.tsx | 66 ++++++++++--------- 2 files changed, 61 insertions(+), 46 deletions(-) diff --git a/frontend/src/components/entity/EntityTable.tsx b/frontend/src/components/entity/EntityTable.tsx index 91a35075c..8cd8c404b 100644 --- a/frontend/src/components/entity/EntityTable.tsx +++ b/frontend/src/components/entity/EntityTable.tsx @@ -11,28 +11,26 @@ import ProjectHooks from "../../api/ProjectHooks.ts"; import { EntityRead } from "../../api/openapi/models/EntityRead.ts"; import { SpanTextRead } from "../../api/openapi/models/SpanTextRead.ts"; -interface EnitityTableRow{ - id: string, - original: EntityRead, +export interface EnitityTableRow extends EntityRead{ + table_id: string, subRows: SpanTextTableRow[]; editable: boolean; } -interface SpanTextTableRow{ - id: string, - original: SpanTextRead, +export interface SpanTextTableRow extends SpanTextRead{ + table_id: string, subRows: SpanTextTableRow[]; editable: boolean; } const columns: MRT_ColumnDef[] = [ { - accessorKey: "original.id", + accessorKey: "id", header: "ID", enableEditing: false, }, { - accessorKey: 'original.name', + accessorKey: 'name', header: 'Name', enableEditing: true, }, @@ -44,6 +42,10 @@ export interface EntityTableActionProps { selectedSpanTexts: SpanTextRead[]; } +export interface EntityTableSaveRowProps extends EntityTableActionProps { + name: string +} + export interface EntityTableProps { projectId: number; // selection @@ -56,6 +58,7 @@ export interface EntityTableProps { renderBottomToolbarCustomActions?: (props: EntityTableActionProps) => React.ReactNode; // editing onSaveEditRow: MRT_TableOptions['onEditingRowSave']; + onCreateSaveRow: (props: EntityTableSaveRowProps) => void; } function EntityTable({ @@ -67,6 +70,7 @@ function EntityTable({ renderTopToolbarCustomActions, renderBottomToolbarCustomActions, onSaveEditRow, + onCreateSaveRow }: EntityTableProps) { // global server state const projectEntities = ProjectHooks.useGetAllEntities(projectId); @@ -85,15 +89,15 @@ function EntityTable({ ); const projectEntitiesRows = projectEntities.data.map(entity => { const subRows = entity.span_texts?.map(span => ({ - id: `S-${span.id}`, - original: {...span, name: span.text}, + ...span, + table_id: `S-${span.id}`, + name: span.text, subRows: [], editable: false, })) || []; - const original = entity; - const id = `E-${entity.id}`; + const table_id = `E-${entity.id}`; const editable = true; - return { id, original, subRows, editable }; + return { table_id, ...entity, subRows, editable }; }); const projectSpanTextMap = projectEntities.data.reduce((acc, entity) => { @@ -112,10 +116,19 @@ function EntityTable({ const table = useMaterialReactTable({ data: projectEntitiesRows, columns: columns, - getRowId: (row) => `${row.id}`, + getRowId: (row) => `${row.table_id}`, enableEditing: (row) => {return row.original.editable}, + createDisplayMode: 'modal', editDisplayMode: 'row', onEditingRowSave: onSaveEditRow, + onCreatingRowSave: (props) => { + onCreateSaveRow({ + selectedEntities: Object.keys(props.table.getState().rowSelection).filter(id => id.startsWith('E-')).map(entityId => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(props.table.getState().rowSelection).filter(id => id.startsWith('S-')).map(spanTextId => projectSpanTextMap[spanTextId]), + name: props.values.name, + table: props.table + }); + }, // style muiTablePaperProps: { elevation: 0, diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx index b2a23e54c..bbd697bb9 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -1,11 +1,11 @@ import { Box, Button, Grid, Portal, Stack, Typography } from "@mui/material"; -import { MRT_RowSelectionState } from "material-react-table"; +import { MRT_RowSelectionState, MRT_TableOptions } from "material-react-table"; import { useContext, useState } from "react"; import { useParams } from "react-router-dom"; import EntityHooks from "../../../api/EntityHooks.ts"; import { EntityRead } from "../../../api/openapi/models/EntityRead.ts"; import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; -import EntityTable from "../../../components/entity/EntityTable.tsx"; +import EntityTable, { EnitityTableRow, EntityTableSaveRowProps, SpanTextTableRow } from "../../../components/entity/EntityTable.tsx"; import { AppBarContext } from "../../../layouts/TwoBarLayout.tsx"; import { useAppSelector } from "../../../plugins/ReduxHooks.ts"; @@ -20,30 +20,12 @@ function EntityDashboard() { const isSplitView = useAppSelector((state) => state.annotatedSegments.isSplitView); const [rowSelectionModel, setRowSelectionModel] = useState({}); const entityMerge = EntityHooks.useMerge(); - const entityResolve = EntityHooks.useResolve(); + const entityRelease = EntityHooks.useRelease(); const entityUpdate = EntityHooks.useUpdateEntity(); - - function handleMerg(selectedEntities: EntityRead[], selectedSpanTexts: SpanTextRead[]): void { - const name = "merge" + selectedEntities[0]?.name + selectedSpanTexts[0]?.text; - const requestBody = { - requestBody: { - name: name, - project_id: projectId, - entity_ids: selectedEntities.map(entity => entity.id), - spantext_ids: selectedSpanTexts.map(spantext => spantext.id) - } - }; - entityMerge.mutate(requestBody); - setRowSelectionModel({}); - } - function handleRelease(selectedEntities: EntityRead[], selectedSpanTexts: SpanTextRead[]): void { - console.log(rowSelectionModel); - console.log(selectedEntities); - console.log(selectedSpanTexts); const requestBody = { requestBody: { project_id: projectId, @@ -51,25 +33,42 @@ function EntityDashboard() { spantext_ids: selectedSpanTexts.map(spantext => spantext.id) } }; - entityResolve.mutate(requestBody); + entityRelease.mutate(requestBody); setRowSelectionModel({}); } - function handleUpdate(props: any): void | Promise { - // TODO fix naming + const handleUpdate: MRT_TableOptions['onEditingRowSave']= async ({ + row, + values, + table, + }) => { + console.log(table) const requestBody = { - entityId: props.values["original.id"], + entityId: row.original.id, requestBody: { - name: props.values["original.name"], - span_text_ids: props.row.original.original.span_texts.map(span_text => span_text.id) + name: values.name, + span_text_ids: row.original.subRows.map(span_text => span_text.id) } }; - console.log(requestBody); entityUpdate.mutate(requestBody); - props.table.setEditingRow(null); + table.setEditingRow(null); + } + function handleMerge(props: EntityTableSaveRowProps): void { + props.table.setCreatingRow(null); + const name = props.name; + const requestBody = { + requestBody: { + name: name, + project_id: projectId, + entity_ids: props.selectedEntities.map(entity => entity.id), + spantext_ids: props.selectedSpanTexts.map(spantext => spantext.id) + } + }; + entityMerge.mutate(requestBody); + setRowSelectionModel({}); } return ( @@ -86,13 +85,16 @@ function EntityDashboard() { projectId={projectId} rowSelectionModel={rowSelectionModel} onRowSelectionChange={setRowSelectionModel} - renderBottomToolbarCustomActions={(props) => ( + renderBottomToolbarCustomActions={(props) => { + return ( - + - )} onSaveEditRow={handleUpdate} /> + )}} + onSaveEditRow={handleUpdate} + onCreateSaveRow={handleMerge}/> From a4eb0870b2cd901ccbc3dfea4b081639377211c0 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Wed, 26 Jun 2024 19:22:42 +0000 Subject: [PATCH 05/18] BugFix span text create_multi now doesnt create duplicates --- backend/src/app/core/data/crud/span_text.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/src/app/core/data/crud/span_text.py b/backend/src/app/core/data/crud/span_text.py index 424eced02..16c687c83 100644 --- a/backend/src/app/core/data/crud/span_text.py +++ b/backend/src/app/core/data/crud/span_text.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Dict, List, Optional from sqlalchemy.orm import Session @@ -25,19 +25,24 @@ def create_multi( # Only create when not already present span_texts: List[SpanTextORM] = [] to_create: List[SpanTextCreate] = [] + span_text_idx: List[int] = [] to_create_idx: List[int] = [] + text_create_map: Dict[str, int] = {} # TODO best would be "insert all (ignore existing) followed by get all" for i, create_dto in enumerate(create_dtos): db_obj = self.read_by_text(db=db, text=create_dto.text) span_texts.append(db_obj) if db_obj is None: - to_create.append(create_dto) - to_create_idx.append(i) + if create_dto.text not in text_create_map: + text_create_map[create_dto.text] = len(to_create) + to_create.append(create_dto) + span_text_idx.append(i) + to_create_idx.append(text_create_map[create_dto.text]) if len(to_create) > 0: created = super().create_multi(db=db, create_dtos=to_create) - for i, obj in zip(to_create_idx, created): - span_texts[i] = obj + for obj_idx, pos_idx in zip(to_create_idx, span_text_idx): + span_texts[pos_idx] = created[obj_idx] # Ignore types: We've made sure that no `None` values remain since we've created # span texts to replace them return span_texts # type: ignore From 615f6f331f936083fb279417d30497e1683800c5 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Wed, 26 Jun 2024 19:25:59 +0000 Subject: [PATCH 06/18] Update entity to include KnowledgeBase and IsHuman (backend) --- backend/src/api/endpoints/entity.py | 4 +++- backend/src/app/core/data/crud/span_annotation.py | 13 +++++++++++-- backend/src/app/core/data/dto/entity.py | 8 ++++++-- .../src/app/core/data/dto/span_text_entity_link.py | 9 +++++++++ backend/src/app/core/data/orm/entity.py | 12 +++++++++++- .../src/app/core/data/orm/span_text_entity_link.py | 3 ++- 6 files changed, 42 insertions(+), 7 deletions(-) diff --git a/backend/src/api/endpoints/entity.py b/backend/src/api/endpoints/entity.py index 5fbe7943b..a2c8c3f7d 100644 --- a/backend/src/api/endpoints/entity.py +++ b/backend/src/api/endpoints/entity.py @@ -35,7 +35,7 @@ def update_by_id( authz_user: AuthzUser = Depends(), ) -> EntityRead: authz_user.assert_in_same_project_as(Crud.ENTITY, entity_id) - + entity.is_human = True db_obj = crud_entity.update(db=db, id=entity_id, update_dto=entity) return EntityRead.model_validate(db_obj) @@ -66,6 +66,8 @@ def merge_entities( name=entity_merge.name, project_id=entity_merge.project_id, span_text_ids=all_span_texts, + is_human=True, + knowledge_base_id=entity_merge.knowledge_base_id, ) db_obj = crud_entity.create(db=db, create_dto=new_entity, force=True) return EntityRead.model_validate(db_obj) diff --git a/backend/src/app/core/data/crud/span_annotation.py b/backend/src/app/core/data/crud/span_annotation.py index 83343ee00..6406ddcb2 100644 --- a/backend/src/app/core/data/crud/span_annotation.py +++ b/backend/src/app/core/data/crud/span_annotation.py @@ -117,13 +117,22 @@ def create_multi( EntityCreate( project_id=project_id, name=dto.span_text, - span_text_ids=[id[0]], + span_text_ids=[id.id], + is_human=False, ) for id, dto in zip(span_texts_orm, create_dtos) ], ) except Exception as e: - raise Exception(e) + raise Exception( + str(e) + + "\n" + + str(span_texts_orm) + + "\n" + + str([type(id) for id in span_texts_orm]) + + "\n" + + str([id.as_dict() for id in span_texts_orm]) + ) # create the SpanAnnotation (and link the SpanText via FK) dto_objs_data = [ diff --git a/backend/src/app/core/data/dto/entity.py b/backend/src/app/core/data/dto/entity.py index fa9d2946b..377e4fa72 100644 --- a/backend/src/app/core/data/dto/entity.py +++ b/backend/src/app/core/data/dto/entity.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List +from typing import List, Optional from pydantic import BaseModel, ConfigDict, Field @@ -10,7 +10,10 @@ # Properties shared across all DTOs class EntityBaseDTO(BaseModel): - pass + is_human: Optional[bool] = Field( + False, description="Whether the link was created by a human" + ) + knowledge_base_id: Optional[str] = Field("", description="Link to wikidata") # Properties for creation @@ -34,6 +37,7 @@ class EntityUpdate(EntityBaseDTO, UpdateDTOBase): # Properties for merging entities/span texts class EntityMerge(EntityBaseDTO): name: str = Field(description="Name of the Entity") + knowledge_base_id: Optional[str] = Field("", description="Link to wikidata") project_id: int = Field(description="Id of the current Project") entity_ids: List[int] = Field(description="List of Entity IDs to merge") spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") diff --git a/backend/src/app/core/data/dto/span_text_entity_link.py b/backend/src/app/core/data/dto/span_text_entity_link.py index 13b7e6b61..8325900b7 100644 --- a/backend/src/app/core/data/dto/span_text_entity_link.py +++ b/backend/src/app/core/data/dto/span_text_entity_link.py @@ -8,6 +8,9 @@ class SpanTextEntityLinkCreate(BaseModel): linked_span_text_id: Optional[int] = Field( description="ID of the linked span text." ) + is_human: Optional[bool] = Field( + False, description="Whether the link was created by a human" + ) class SpanTextEntityLinkUpdate(BaseModel): @@ -15,6 +18,9 @@ class SpanTextEntityLinkUpdate(BaseModel): linked_span_text_id: Optional[int] = Field( description="ID of the linked span text." ) + is_human: Optional[bool] = Field( + False, description="Whether the link was created by a human" + ) class SpanTextEntityLinkRead(BaseModel): @@ -24,3 +30,6 @@ class SpanTextEntityLinkRead(BaseModel): description="ID of the linked span text." ) model_config = ConfigDict(from_attributes=True) + is_human: Optional[bool] = Field( + False, description="Whether the link was created by a human" + ) diff --git a/backend/src/app/core/data/orm/entity.py b/backend/src/app/core/data/orm/entity.py index 5608f91d5..07cc92dba 100644 --- a/backend/src/app/core/data/orm/entity.py +++ b/backend/src/app/core/data/orm/entity.py @@ -1,7 +1,15 @@ from datetime import datetime from typing import TYPE_CHECKING, List, Optional -from sqlalchemy import DateTime, ForeignKey, Integer, String, UniqueConstraint, func +from sqlalchemy import ( + Boolean, + DateTime, + ForeignKey, + Integer, + String, + UniqueConstraint, + func, +) from sqlalchemy.orm import Mapped, mapped_column, relationship from app.core.data.orm.orm_base import ORMBase @@ -19,6 +27,8 @@ class EntityORM(ORMBase): updated: Mapped[Optional[datetime]] = mapped_column( DateTime, server_default=func.now(), onupdate=func.current_timestamp() ) + is_human: Mapped[Boolean] = mapped_column(Boolean, default=False, index=True) + knowledge_base_id: Mapped[str] = mapped_column(String, default="", index=True) project_id: Mapped[int] = mapped_column( Integer, diff --git a/backend/src/app/core/data/orm/span_text_entity_link.py b/backend/src/app/core/data/orm/span_text_entity_link.py index ce3c0d017..086f0d9f6 100644 --- a/backend/src/app/core/data/orm/span_text_entity_link.py +++ b/backend/src/app/core/data/orm/span_text_entity_link.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Optional -from sqlalchemy import ForeignKey, Integer +from sqlalchemy import Boolean, ForeignKey, Integer from sqlalchemy.orm import Mapped, mapped_column from app.core.data.orm.orm_base import ORMBase @@ -17,3 +17,4 @@ class SpanTextEntityLinkORM(ORMBase): linked_span_text_id: Mapped[Optional[int]] = mapped_column( Integer, ForeignKey("spantext.id", ondelete="CASCADE"), index=True ) + is_human: Mapped[Boolean] = mapped_column(Boolean, default=False, index=True) From 2340280ebda6d04a3c48aa494de4913d67986bd6 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Wed, 26 Jun 2024 19:27:37 +0000 Subject: [PATCH 07/18] Update entity to include KnowledgeBase and IsHuman (frontend) --- frontend/src/components/entity/EntityTable.tsx | 18 +++++++++++++++--- .../EntityDashboard/EntityDashboard.tsx | 11 +++++++---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/entity/EntityTable.tsx b/frontend/src/components/entity/EntityTable.tsx index 8cd8c404b..f223efc6d 100644 --- a/frontend/src/components/entity/EntityTable.tsx +++ b/frontend/src/components/entity/EntityTable.tsx @@ -1,4 +1,5 @@ import { + LiteralUnion, MRT_ColumnDef, MRT_RowSelectionState, MRT_TableInstance, @@ -34,6 +35,17 @@ const columns: MRT_ColumnDef[] = [ header: 'Name', enableEditing: true, }, + { + accessorKey: 'knowledge_base_id', + header: 'Knowledge Base ID', + enableEditing: true, + }, + { + accessorKey: 'is_human', + header: 'Is Human', + enableEditing: false, + Cell: ({ cell }) => {console.log(cell); return cell.getValue() ? 'True' : 'False';}, + }, ]; export interface EntityTableActionProps { @@ -43,7 +55,7 @@ export interface EntityTableActionProps { } export interface EntityTableSaveRowProps extends EntityTableActionProps { - name: string + values: Record, string > } export interface EntityTableProps { @@ -78,7 +90,7 @@ function EntityTable({ // computed const { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap } = useMemo(() => { if (!projectEntities.data) return { projectEntitiesMap: {} as Record, projectEntitiesRows: [], projectSpanTextMap: {} as Record}; - + const projectEntitiesMap = projectEntities.data.reduce( (entity_map, projectEntity) => { const id = `E-${projectEntity.id}`; @@ -125,7 +137,7 @@ function EntityTable({ onCreateSaveRow({ selectedEntities: Object.keys(props.table.getState().rowSelection).filter(id => id.startsWith('E-')).map(entityId => projectEntitiesMap[entityId]), selectedSpanTexts: Object.keys(props.table.getState().rowSelection).filter(id => id.startsWith('S-')).map(spanTextId => projectSpanTextMap[spanTextId]), - name: props.values.name, + values: props.values, table: props.table }); }, diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx index bbd697bb9..29966f700 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -44,27 +44,30 @@ function EntityDashboard() { values, table, }) => { - console.log(table) const requestBody = { entityId: row.original.id, requestBody: { name: values.name, - span_text_ids: row.original.subRows.map(span_text => span_text.id) + span_text_ids: row.original.subRows.map(span_text => span_text.id), + knowledge_base_id: values.knowledge_base_id } }; entityUpdate.mutate(requestBody); table.setEditingRow(null); } + function handleMerge(props: EntityTableSaveRowProps): void { props.table.setCreatingRow(null); - const name = props.name; + const name = props.values.name; + const knowledge_base_id = props.values.knowledge_base_id; const requestBody = { requestBody: { name: name, project_id: projectId, entity_ids: props.selectedEntities.map(entity => entity.id), - spantext_ids: props.selectedSpanTexts.map(spantext => spantext.id) + spantext_ids: props.selectedSpanTexts.map(spantext => spantext.id), + knowledge_base_id: knowledge_base_id } }; entityMerge.mutate(requestBody); From 2b93d9a2f45fc5ee713ba145a0e9cfcfd22d8b7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 27 Jun 2024 12:56:30 +0000 Subject: [PATCH 08/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- backend/src/app/core/data/crud/entity.py | 2 +- backend/src/app/core/data/orm/entity.py | 1 - frontend/src/api/EntityHooks.ts | 28 ++-- .../src/components/entity/EntityTable.tsx | 124 +++++++++++------- .../EntityDashboard/EntityDashboard.tsx | 59 +++++---- 5 files changed, 121 insertions(+), 93 deletions(-) diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py index 0946ff36e..68573627e 100644 --- a/backend/src/app/core/data/crud/entity.py +++ b/backend/src/app/core/data/crud/entity.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List from fastapi.encoders import jsonable_encoder from sqlalchemy import select diff --git a/backend/src/app/core/data/orm/entity.py b/backend/src/app/core/data/orm/entity.py index 07cc92dba..9bea18ad0 100644 --- a/backend/src/app/core/data/orm/entity.py +++ b/backend/src/app/core/data/orm/entity.py @@ -7,7 +7,6 @@ ForeignKey, Integer, String, - UniqueConstraint, func, ) from sqlalchemy.orm import Mapped, mapped_column, relationship diff --git a/frontend/src/api/EntityHooks.ts b/frontend/src/api/EntityHooks.ts index 41c8fb3d3..8501d888e 100644 --- a/frontend/src/api/EntityHooks.ts +++ b/frontend/src/api/EntityHooks.ts @@ -4,7 +4,6 @@ import { QueryKey } from "./QueryKey.ts"; import { EntityRead } from "./openapi/models/EntityRead.ts"; import { EntityService } from "./openapi/services/EntityService.ts"; - // enitity const useGetEntity = (entityId: number | null | undefined) => useQuery({ @@ -43,20 +42,19 @@ const useMerge = () => useMutation({ mutationFn: EntityService.mergeEntities, onSuccess: (data) => { - queryClient.invalidateQueries({queryKey: [QueryKey.ENTITY, data.project_id]}); - queryClient.invalidateQueries({queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id]}) - } - }) - - const useResolve = () => - useMutation({ - mutationFn: EntityService.resolveEntities, - onSuccess: (data) => { - queryClient.invalidateQueries({queryKey: [QueryKey.ENTITY, data[0].project_id]}); - queryClient.invalidateQueries({queryKey: [QueryKey.PROJECT_ENTITIES, data[0].project_id]}) - } - }) + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data.project_id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id] }); + }, + }); +const useResolve = () => + useMutation({ + mutationFn: EntityService.resolveEntities, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data[0].project_id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data[0].project_id] }); + }, + }); const EntityHooks = { useGetEntity, @@ -64,7 +62,7 @@ const EntityHooks = { useUpdateEntity, useDeleteEntity, useMerge, - useResolve + useResolve, }; export default EntityHooks; diff --git a/frontend/src/components/entity/EntityTable.tsx b/frontend/src/components/entity/EntityTable.tsx index f223efc6d..3a09ddf1e 100644 --- a/frontend/src/components/entity/EntityTable.tsx +++ b/frontend/src/components/entity/EntityTable.tsx @@ -12,39 +12,42 @@ import ProjectHooks from "../../api/ProjectHooks.ts"; import { EntityRead } from "../../api/openapi/models/EntityRead.ts"; import { SpanTextRead } from "../../api/openapi/models/SpanTextRead.ts"; -export interface EnitityTableRow extends EntityRead{ - table_id: string, +export interface EnitityTableRow extends EntityRead { + table_id: string; subRows: SpanTextTableRow[]; editable: boolean; } -export interface SpanTextTableRow extends SpanTextRead{ - table_id: string, +export interface SpanTextTableRow extends SpanTextRead { + table_id: string; subRows: SpanTextTableRow[]; editable: boolean; } -const columns: MRT_ColumnDef[] = [ +const columns: MRT_ColumnDef[] = [ { accessorKey: "id", header: "ID", enableEditing: false, }, { - accessorKey: 'name', - header: 'Name', + accessorKey: "name", + header: "Name", enableEditing: true, }, { - accessorKey: 'knowledge_base_id', - header: 'Knowledge Base ID', + accessorKey: "knowledge_base_id", + header: "Knowledge Base ID", enableEditing: true, }, { - accessorKey: 'is_human', - header: 'Is Human', + accessorKey: "is_human", + header: "Is Human", enableEditing: false, - Cell: ({ cell }) => {console.log(cell); return cell.getValue() ? 'True' : 'False';}, + Cell: ({ cell }) => { + console.log(cell); + return cell.getValue() ? "True" : "False"; + }, }, ]; @@ -55,7 +58,7 @@ export interface EntityTableActionProps { } export interface EntityTableSaveRowProps extends EntityTableActionProps { - values: Record, string > + values: Record, string>; } export interface EntityTableProps { @@ -69,7 +72,7 @@ export interface EntityTableProps { renderTopToolbarCustomActions?: (props: EntityTableActionProps) => React.ReactNode; renderBottomToolbarCustomActions?: (props: EntityTableActionProps) => React.ReactNode; // editing - onSaveEditRow: MRT_TableOptions['onEditingRowSave']; + onSaveEditRow: MRT_TableOptions["onEditingRowSave"]; onCreateSaveRow: (props: EntityTableSaveRowProps) => void; } @@ -82,14 +85,19 @@ function EntityTable({ renderTopToolbarCustomActions, renderBottomToolbarCustomActions, onSaveEditRow, - onCreateSaveRow + onCreateSaveRow, }: EntityTableProps) { // global server state const projectEntities = ProjectHooks.useGetAllEntities(projectId); // computed const { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap } = useMemo(() => { - if (!projectEntities.data) return { projectEntitiesMap: {} as Record, projectEntitiesRows: [], projectSpanTextMap: {} as Record}; + if (!projectEntities.data) + return { + projectEntitiesMap: {} as Record, + projectEntitiesRows: [], + projectSpanTextMap: {} as Record, + }; const projectEntitiesMap = projectEntities.data.reduce( (entity_map, projectEntity) => { @@ -99,46 +107,56 @@ function EntityTable({ }, {} as Record, ); - const projectEntitiesRows = projectEntities.data.map(entity => { - const subRows = entity.span_texts?.map(span => ({ - ...span, - table_id: `S-${span.id}`, - name: span.text, - subRows: [], - editable: false, - })) || []; + const projectEntitiesRows = projectEntities.data.map((entity) => { + const subRows = + entity.span_texts?.map((span) => ({ + ...span, + table_id: `S-${span.id}`, + name: span.text, + subRows: [], + editable: false, + })) || []; const table_id = `E-${entity.id}`; const editable = true; return { table_id, ...entity, subRows, editable }; }); - const projectSpanTextMap = projectEntities.data.reduce((acc, entity) => { - if (Array.isArray(entity.span_texts)) { - entity.span_texts.forEach(span => { - acc[`S-${span.id}`] = span; - }); - } - return acc; - }, {} as Record) - - return { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap}; + const projectSpanTextMap = projectEntities.data.reduce( + (acc, entity) => { + if (Array.isArray(entity.span_texts)) { + entity.span_texts.forEach((span) => { + acc[`S-${span.id}`] = span; + }); + } + return acc; + }, + {} as Record, + ); + + return { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap }; }, [projectEntities.data]); // table - const table = useMaterialReactTable({ + const table = useMaterialReactTable({ data: projectEntitiesRows, columns: columns, getRowId: (row) => `${row.table_id}`, - enableEditing: (row) => {return row.original.editable}, - createDisplayMode: 'modal', - editDisplayMode: 'row', + enableEditing: (row) => { + return row.original.editable; + }, + createDisplayMode: "modal", + editDisplayMode: "row", onEditingRowSave: onSaveEditRow, onCreatingRowSave: (props) => { onCreateSaveRow({ - selectedEntities: Object.keys(props.table.getState().rowSelection).filter(id => id.startsWith('E-')).map(entityId => projectEntitiesMap[entityId]), - selectedSpanTexts: Object.keys(props.table.getState().rowSelection).filter(id => id.startsWith('S-')).map(spanTextId => projectSpanTextMap[spanTextId]), + selectedEntities: Object.keys(props.table.getState().rowSelection) + .filter((id) => id.startsWith("E-")) + .map((entityId) => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(props.table.getState().rowSelection) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]), values: props.values, - table: props.table + table: props.table, }); }, // style @@ -176,24 +194,36 @@ function EntityTable({ ? (props) => renderTopToolbarCustomActions({ table: props.table, - selectedEntities: Object.keys(rowSelectionModel).filter(id => id.startsWith('E-')).map((entityId) => projectEntitiesMap[entityId]), - selectedSpanTexts: Object.keys(rowSelectionModel).filter(id => id.startsWith('S-')).map((spanTextId) => projectSpanTextMap[spanTextId]), + selectedEntities: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("E-")) + .map((entityId) => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]), }) : undefined, renderToolbarInternalActions: renderToolbarInternalActions ? (props) => renderToolbarInternalActions({ table: props.table, - selectedEntities: Object.keys(rowSelectionModel).filter(id => id.startsWith('E-')).map((entityId) => projectEntitiesMap[entityId]), - selectedSpanTexts: Object.keys(rowSelectionModel).filter(id => id.startsWith('S-')).map((spanTextId) => projectSpanTextMap[spanTextId]), + selectedEntities: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("E-")) + .map((entityId) => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]), }) : undefined, renderBottomToolbarCustomActions: renderBottomToolbarCustomActions ? (props) => renderBottomToolbarCustomActions({ table: props.table, - selectedEntities: Object.keys(rowSelectionModel).filter(id => id.startsWith('E-')).map((entityId) => projectEntitiesMap[entityId]), - selectedSpanTexts: Object.keys(rowSelectionModel).filter(id => id.startsWith('S-')).map((spanTextId) => projectSpanTextMap[spanTextId]), + selectedEntities: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("E-")) + .map((entityId) => projectEntitiesMap[entityId]), + selectedSpanTexts: Object.keys(rowSelectionModel) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]), }) : undefined, // hide columns per default diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx index 29966f700..444ca195f 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -5,11 +5,14 @@ import { useParams } from "react-router-dom"; import EntityHooks from "../../../api/EntityHooks.ts"; import { EntityRead } from "../../../api/openapi/models/EntityRead.ts"; import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; -import EntityTable, { EnitityTableRow, EntityTableSaveRowProps, SpanTextTableRow } from "../../../components/entity/EntityTable.tsx"; +import EntityTable, { + EnitityTableRow, + EntityTableSaveRowProps, + SpanTextTableRow, +} from "../../../components/entity/EntityTable.tsx"; import { AppBarContext } from "../../../layouts/TwoBarLayout.tsx"; import { useAppSelector } from "../../../plugins/ReduxHooks.ts"; - function EntityDashboard() { const appBarContainerRef = useContext(AppBarContext); @@ -23,39 +26,34 @@ function EntityDashboard() { const entityRelease = EntityHooks.useRelease(); const entityUpdate = EntityHooks.useUpdateEntity(); - - function handleRelease(selectedEntities: EntityRead[], selectedSpanTexts: SpanTextRead[]): void { const requestBody = { requestBody: { project_id: projectId, - entity_ids: selectedEntities.map(entity => entity.id), - spantext_ids: selectedSpanTexts.map(spantext => spantext.id) - } + entity_ids: selectedEntities.map((entity) => entity.id), + spantext_ids: selectedSpanTexts.map((spantext) => spantext.id), + }, }; entityRelease.mutate(requestBody); setRowSelectionModel({}); } - - - const handleUpdate: MRT_TableOptions['onEditingRowSave']= async ({ + const handleUpdate: MRT_TableOptions["onEditingRowSave"] = async ({ row, values, table, }) => { - const requestBody = - { + const requestBody = { entityId: row.original.id, requestBody: { name: values.name, - span_text_ids: row.original.subRows.map(span_text => span_text.id), - knowledge_base_id: values.knowledge_base_id - } + span_text_ids: row.original.subRows.map((span_text) => span_text.id), + knowledge_base_id: values.knowledge_base_id, + }, }; entityUpdate.mutate(requestBody); table.setEditingRow(null); - } + }; function handleMerge(props: EntityTableSaveRowProps): void { props.table.setCreatingRow(null); @@ -65,10 +63,10 @@ function EntityDashboard() { requestBody: { name: name, project_id: projectId, - entity_ids: props.selectedEntities.map(entity => entity.id), - spantext_ids: props.selectedSpanTexts.map(spantext => spantext.id), - knowledge_base_id: knowledge_base_id - } + entity_ids: props.selectedEntities.map((entity) => entity.id), + spantext_ids: props.selectedSpanTexts.map((spantext) => spantext.id), + knowledge_base_id: knowledge_base_id, + }, }; entityMerge.mutate(requestBody); setRowSelectionModel({}); @@ -83,21 +81,24 @@ function EntityDashboard() { - { - return ( - - - - - - )}} + return ( + + + + + + ); + }} onSaveEditRow={handleUpdate} - onCreateSaveRow={handleMerge}/> + onCreateSaveRow={handleMerge} + /> From 02ac98af423d28421e47c257ba7cfa1e2aa9b305 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Thu, 27 Jun 2024 16:23:29 +0000 Subject: [PATCH 09/18] update db and openapi --- ...31_add_entity_and_span_text_entity_link.py | 112 ++++++++ frontend/src/api/EntityHooks.ts | 36 +-- .../src/api/openapi/models/EntityMerge.ts | 30 +++ frontend/src/api/openapi/models/EntityRead.ts | 39 +++ .../src/api/openapi/models/EntityRelease.ts | 26 ++ .../src/api/openapi/models/EntityUpdate.ts | 22 ++ .../src/api/openapi/models/SpanTextRead.ts | 14 + .../src/api/openapi/services/EntityService.ts | 70 +++++ .../api/openapi/services/ProjectService.ts | 18 ++ frontend/src/openapi.json | 247 ++++++++++++++++++ 10 files changed, 582 insertions(+), 32 deletions(-) create mode 100644 backend/src/alembic/versions/13cc78f77731_add_entity_and_span_text_entity_link.py create mode 100644 frontend/src/api/openapi/models/EntityMerge.ts create mode 100644 frontend/src/api/openapi/models/EntityRead.ts create mode 100644 frontend/src/api/openapi/models/EntityRelease.ts create mode 100644 frontend/src/api/openapi/models/EntityUpdate.ts create mode 100644 frontend/src/api/openapi/models/SpanTextRead.ts create mode 100644 frontend/src/api/openapi/services/EntityService.ts diff --git a/backend/src/alembic/versions/13cc78f77731_add_entity_and_span_text_entity_link.py b/backend/src/alembic/versions/13cc78f77731_add_entity_and_span_text_entity_link.py new file mode 100644 index 000000000..be96c1946 --- /dev/null +++ b/backend/src/alembic/versions/13cc78f77731_add_entity_and_span_text_entity_link.py @@ -0,0 +1,112 @@ +"""add entity and span text entity link + +Revision ID: 13cc78f77731 +Revises: 2b91203d1bb6 +Create Date: 2024-06-27 16:05:14.589423 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "13cc78f77731" +down_revision: Union[str, None] = "2b91203d1bb6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "entity", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column( + "created", sa.DateTime(), server_default=sa.text("now()"), nullable=True + ), + sa.Column( + "updated", sa.DateTime(), server_default=sa.text("now()"), nullable=True + ), + sa.Column("is_human", sa.Boolean(), nullable=False), + sa.Column("knowledge_base_id", sa.String(), nullable=False), + sa.Column("project_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index(op.f("ix_entity_created"), "entity", ["created"], unique=False) + op.create_index(op.f("ix_entity_id"), "entity", ["id"], unique=False) + op.create_index(op.f("ix_entity_is_human"), "entity", ["is_human"], unique=False) + op.create_index( + op.f("ix_entity_knowledge_base_id"), + "entity", + ["knowledge_base_id"], + unique=False, + ) + op.create_index(op.f("ix_entity_name"), "entity", ["name"], unique=False) + op.create_index( + op.f("ix_entity_project_id"), "entity", ["project_id"], unique=False + ) + op.create_table( + "spantextentitylink", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("linked_entity_id", sa.Integer(), nullable=True), + sa.Column("linked_span_text_id", sa.Integer(), nullable=True), + sa.Column("is_human", sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint( + ["linked_entity_id"], ["entity.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["linked_span_text_id"], ["spantext.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_spantextentitylink_id"), "spantextentitylink", ["id"], unique=False + ) + op.create_index( + op.f("ix_spantextentitylink_is_human"), + "spantextentitylink", + ["is_human"], + unique=False, + ) + op.create_index( + op.f("ix_spantextentitylink_linked_entity_id"), + "spantextentitylink", + ["linked_entity_id"], + unique=False, + ) + op.create_index( + op.f("ix_spantextentitylink_linked_span_text_id"), + "spantextentitylink", + ["linked_span_text_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f("ix_spantextentitylink_linked_span_text_id"), + table_name="spantextentitylink", + ) + op.drop_index( + op.f("ix_spantextentitylink_linked_entity_id"), table_name="spantextentitylink" + ) + op.drop_index( + op.f("ix_spantextentitylink_is_human"), table_name="spantextentitylink" + ) + op.drop_index(op.f("ix_spantextentitylink_id"), table_name="spantextentitylink") + op.drop_table("spantextentitylink") + op.drop_index(op.f("ix_entity_project_id"), table_name="entity") + op.drop_index(op.f("ix_entity_name"), table_name="entity") + op.drop_index(op.f("ix_entity_knowledge_base_id"), table_name="entity") + op.drop_index(op.f("ix_entity_is_human"), table_name="entity") + op.drop_index(op.f("ix_entity_id"), table_name="entity") + op.drop_index(op.f("ix_entity_created"), table_name="entity") + op.drop_table("entity") + # ### end Alembic commands ### diff --git a/frontend/src/api/EntityHooks.ts b/frontend/src/api/EntityHooks.ts index 41c8fb3d3..243471faf 100644 --- a/frontend/src/api/EntityHooks.ts +++ b/frontend/src/api/EntityHooks.ts @@ -1,25 +1,9 @@ -import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMutation } from "@tanstack/react-query"; import queryClient from "../plugins/ReactQueryClient.ts"; import { QueryKey } from "./QueryKey.ts"; -import { EntityRead } from "./openapi/models/EntityRead.ts"; import { EntityService } from "./openapi/services/EntityService.ts"; -// enitity -const useGetEntity = (entityId: number | null | undefined) => - useQuery({ - queryKey: [QueryKey.ENTITY, entityId], - queryFn: () => EntityService.getById({ entityId: entityId! }), - enabled: !!entityId, - }); - -const useCreateEntity = () => - useMutation({ - mutationFn: EntityService.createNewEntity, - onSuccess: (newEntity, variables) => { - queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, variables.requestBody.project_id] }); - }, - }); const useUpdateEntity = () => useMutation({ @@ -30,15 +14,6 @@ const useUpdateEntity = () => }, }); -const useDeleteEntity = () => - useMutation({ - mutationFn: EntityService.deleteById, - onSuccess: (data) => { - queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data.id] }); - queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id] }); - }, - }); - const useMerge = () => useMutation({ mutationFn: EntityService.mergeEntities, @@ -48,9 +23,9 @@ const useMerge = () => } }) - const useResolve = () => + const useRelease = () => useMutation({ - mutationFn: EntityService.resolveEntities, + mutationFn: EntityService.releaseEntities, onSuccess: (data) => { queryClient.invalidateQueries({queryKey: [QueryKey.ENTITY, data[0].project_id]}); queryClient.invalidateQueries({queryKey: [QueryKey.PROJECT_ENTITIES, data[0].project_id]}) @@ -59,12 +34,9 @@ const useMerge = () => const EntityHooks = { - useGetEntity, - useCreateEntity, useUpdateEntity, - useDeleteEntity, useMerge, - useResolve + useRelease }; export default EntityHooks; diff --git a/frontend/src/api/openapi/models/EntityMerge.ts b/frontend/src/api/openapi/models/EntityMerge.ts new file mode 100644 index 000000000..e8ad9fd81 --- /dev/null +++ b/frontend/src/api/openapi/models/EntityMerge.ts @@ -0,0 +1,30 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EntityMerge = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * Name of the Entity + */ + name: string; + /** + * Id of the current Project + */ + project_id: number; + /** + * List of Entity IDs to merge + */ + entity_ids: Array; + /** + * List of Span Text IDs to merge + */ + spantext_ids: Array; +}; diff --git a/frontend/src/api/openapi/models/EntityRead.ts b/frontend/src/api/openapi/models/EntityRead.ts new file mode 100644 index 000000000..fdc41e328 --- /dev/null +++ b/frontend/src/api/openapi/models/EntityRead.ts @@ -0,0 +1,39 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { SpanTextRead } from "./SpanTextRead"; +export type EntityRead = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * ID of the Entity + */ + id: number; + /** + * Name of the Entity + */ + name: string; + /** + * Project the Entity belongs to + */ + project_id: number; + /** + * Created timestamp of the Entity + */ + created: string; + /** + * Updated timestamp of the Entity + */ + updated: string; + /** + * The SpanTexts belonging to this entity + */ + span_texts?: Array; +}; diff --git a/frontend/src/api/openapi/models/EntityRelease.ts b/frontend/src/api/openapi/models/EntityRelease.ts new file mode 100644 index 000000000..9dbf3b8a7 --- /dev/null +++ b/frontend/src/api/openapi/models/EntityRelease.ts @@ -0,0 +1,26 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EntityRelease = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * Id of the current Project + */ + project_id: number; + /** + * List of Entity IDs to merge + */ + entity_ids: Array; + /** + * List of Span Text IDs to merge + */ + spantext_ids: Array; +}; diff --git a/frontend/src/api/openapi/models/EntityUpdate.ts b/frontend/src/api/openapi/models/EntityUpdate.ts new file mode 100644 index 000000000..3bc9d4214 --- /dev/null +++ b/frontend/src/api/openapi/models/EntityUpdate.ts @@ -0,0 +1,22 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EntityUpdate = { + /** + * Whether the link was created by a human + */ + is_human?: boolean | null; + /** + * Link to wikidata + */ + knowledge_base_id?: string | null; + /** + * Name of the Entity + */ + name: string; + /** + * Span Text Ids which belong to this Entity + */ + span_text_ids: Array; +}; diff --git a/frontend/src/api/openapi/models/SpanTextRead.ts b/frontend/src/api/openapi/models/SpanTextRead.ts new file mode 100644 index 000000000..c30d39e4a --- /dev/null +++ b/frontend/src/api/openapi/models/SpanTextRead.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type SpanTextRead = { + /** + * Code of the SpanText + */ + text?: string; + /** + * ID of the SpanText + */ + id: number; +}; diff --git a/frontend/src/api/openapi/services/EntityService.ts b/frontend/src/api/openapi/services/EntityService.ts new file mode 100644 index 000000000..ba0317fa5 --- /dev/null +++ b/frontend/src/api/openapi/services/EntityService.ts @@ -0,0 +1,70 @@ +/* generated using openapi-typescript-codegen -- do no edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { EntityMerge } from "../models/EntityMerge"; +import type { EntityRead } from "../models/EntityRead"; +import type { EntityRelease } from "../models/EntityRelease"; +import type { EntityUpdate } from "../models/EntityUpdate"; +import type { CancelablePromise } from "../core/CancelablePromise"; +import { OpenAPI } from "../core/OpenAPI"; +import { request as __request } from "../core/request"; +export class EntityService { + /** + * Updates the Entity with the given ID. + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static updateById({ + entityId, + requestBody, + }: { + entityId: number; + requestBody: EntityUpdate; + }): CancelablePromise { + return __request(OpenAPI, { + method: "PATCH", + url: "/entity/{entity_id}", + path: { + entity_id: entityId, + }, + body: requestBody, + mediaType: "application/json", + errors: { + 422: `Validation Error`, + }, + }); + } + /** + * Merges entities and/or span texts with given IDs. + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static mergeEntities({ requestBody }: { requestBody: EntityMerge }): CancelablePromise { + return __request(OpenAPI, { + method: "PUT", + url: "/entity/merge", + body: requestBody, + mediaType: "application/json", + errors: { + 422: `Validation Error`, + }, + }); + } + /** + * Releases entities and/or span texts with given IDs. + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static releaseEntities({ requestBody }: { requestBody: EntityRelease }): CancelablePromise> { + return __request(OpenAPI, { + method: "PUT", + url: "/entity/release", + body: requestBody, + mediaType: "application/json", + errors: { + 422: `Validation Error`, + }, + }); + } +} diff --git a/frontend/src/api/openapi/services/ProjectService.ts b/frontend/src/api/openapi/services/ProjectService.ts index 43a210b4a..3d07da016 100644 --- a/frontend/src/api/openapi/services/ProjectService.ts +++ b/frontend/src/api/openapi/services/ProjectService.ts @@ -7,6 +7,7 @@ import type { ActionRead } from "../models/ActionRead"; import type { Body_project_upload_project_sdoc } from "../models/Body_project_upload_project_sdoc"; import type { CodeRead } from "../models/CodeRead"; import type { DocumentTagRead } from "../models/DocumentTagRead"; +import type { EntityRead } from "../models/EntityRead"; import type { MemoCreate } from "../models/MemoCreate"; import type { MemoRead } from "../models/MemoRead"; import type { PreprocessingJobRead } from "../models/PreprocessingJobRead"; @@ -525,4 +526,21 @@ export class ProjectService { }, }); } + /** + * Returns all Entities of the Project with the given ID + * @returns EntityRead Successful Response + * @throws ApiError + */ + public static getProjectEntities({ projId }: { projId: number }): CancelablePromise> { + return __request(OpenAPI, { + method: "GET", + url: "/project/{proj_id}/entity", + path: { + proj_id: projId, + }, + errors: { + 422: `Validation Error`, + }, + }); + } } diff --git a/frontend/src/openapi.json b/frontend/src/openapi.json index 79a8de166..b04831d6f 100644 --- a/frontend/src/openapi.json +++ b/frontend/src/openapi.json @@ -951,6 +951,35 @@ } } }, + "/project/{proj_id}/entity": { + "get": { + "tags": ["project"], + "summary": "Returns all Entities of the Project with the given ID", + "operationId": "get_project_entities", + "security": [{ "OAuth2PasswordBearer": [] }], + "parameters": [ + { "name": "proj_id", "in": "path", "required": true, "schema": { "type": "integer", "title": "Proj Id" } } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { "$ref": "#/components/schemas/EntityRead" }, + "title": "Response Project-Get Project Entities" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + } + } + }, "/sdoc/{sdoc_id}": { "get": { "tags": ["sourceDocument"], @@ -3001,6 +3030,83 @@ } } }, + "/entity/{entity_id}": { + "patch": { + "tags": ["entity"], + "summary": "Updates the Entity with the given ID.", + "operationId": "update_by_id", + "security": [{ "OAuth2PasswordBearer": [] }], + "parameters": [ + { "name": "entity_id", "in": "path", "required": true, "schema": { "type": "integer", "title": "Entity Id" } } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityUpdate" } } } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityRead" } } } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + } + } + }, + "/entity/merge": { + "put": { + "tags": ["entity"], + "summary": "Merges entities and/or span texts with given IDs.", + "operationId": "merge_entities", + "requestBody": { + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityMerge" } } }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityRead" } } } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + }, + "security": [{ "OAuth2PasswordBearer": [] }] + } + }, + "/entity/release": { + "put": { + "tags": ["entity"], + "summary": "Releases entities and/or span texts with given IDs.", + "operationId": "release_entities", + "requestBody": { + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EntityRelease" } } }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { "$ref": "#/components/schemas/EntityRead" }, + "type": "array", + "title": "Response Entity-Release Entities" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } + } + }, + "security": [{ "OAuth2PasswordBearer": [] }] + } + }, "/memo/{memo_id}": { "get": { "tags": ["memo"], @@ -6812,6 +6918,138 @@ "required": ["document_id"], "title": "ElasticSearchDocumentHit" }, + "EntityMerge": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "name": { "type": "string", "title": "Name", "description": "Name of the Entity" }, + "project_id": { "type": "integer", "title": "Project Id", "description": "Id of the current Project" }, + "entity_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Entity Ids", + "description": "List of Entity IDs to merge" + }, + "spantext_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Spantext Ids", + "description": "List of Span Text IDs to merge" + } + }, + "type": "object", + "required": ["name", "project_id", "entity_ids", "spantext_ids"], + "title": "EntityMerge" + }, + "EntityRead": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "id": { "type": "integer", "title": "Id", "description": "ID of the Entity" }, + "name": { "type": "string", "title": "Name", "description": "Name of the Entity" }, + "project_id": { "type": "integer", "title": "Project Id", "description": "Project the Entity belongs to" }, + "created": { + "type": "string", + "format": "date-time", + "title": "Created", + "description": "Created timestamp of the Entity" + }, + "updated": { + "type": "string", + "format": "date-time", + "title": "Updated", + "description": "Updated timestamp of the Entity" + }, + "span_texts": { + "items": { "$ref": "#/components/schemas/SpanTextRead" }, + "type": "array", + "title": "Span Texts", + "description": "The SpanTexts belonging to this entity", + "default": [] + } + }, + "type": "object", + "required": ["id", "name", "project_id", "created", "updated"], + "title": "EntityRead" + }, + "EntityRelease": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "project_id": { "type": "integer", "title": "Project Id", "description": "Id of the current Project" }, + "entity_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Entity Ids", + "description": "List of Entity IDs to merge" + }, + "spantext_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Spantext Ids", + "description": "List of Span Text IDs to merge" + } + }, + "type": "object", + "required": ["project_id", "entity_ids", "spantext_ids"], + "title": "EntityRelease" + }, + "EntityUpdate": { + "properties": { + "is_human": { + "anyOf": [{ "type": "boolean" }, { "type": "null" }], + "title": "Is Human", + "description": "Whether the link was created by a human", + "default": false + }, + "knowledge_base_id": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "title": "Knowledge Base Id", + "description": "Link to wikidata", + "default": "" + }, + "name": { "type": "string", "title": "Name", "description": "Name of the Entity" }, + "span_text_ids": { + "items": { "type": "integer" }, + "type": "array", + "title": "Span Text Ids", + "description": "Span Text Ids which belong to this Entity" + } + }, + "type": "object", + "required": ["name", "span_text_ids"], + "title": "EntityUpdate" + }, "ExportFormat": { "type": "string", "enum": ["CSV", "JSON"], "title": "ExportFormat" }, "ExportJobParameters": { "properties": { @@ -8617,6 +8855,15 @@ "type": "object", "title": "SpanGroupUpdate" }, + "SpanTextRead": { + "properties": { + "text": { "type": "string", "title": "Text", "description": "Code of the SpanText" }, + "id": { "type": "integer", "title": "Id", "description": "ID of the SpanText" } + }, + "type": "object", + "required": ["id"], + "title": "SpanTextRead" + }, "StringOperator": { "type": "string", "enum": ["STRING_CONTAINS", "STRING_EQUALS", "STRING_NOT_EQUALS", "STRING_STARTS_WITH", "STRING_ENDS_WITH"], From a03a8959be48dac06f6d9e300493c56f8d17e34f Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Thu, 4 Jul 2024 14:41:13 +0000 Subject: [PATCH 10/18] update, more efficient entity cleanup, cleanup enpoint code --- backend/src/api/endpoints/entity.py | 47 +------- backend/src/app/core/data/crud/entity.py | 102 +++++++++++++++--- .../src/app/core/data/crud/span_annotation.py | 7 +- .../core/data/crud/span_text_entity_link.py | 39 ++++--- 4 files changed, 120 insertions(+), 75 deletions(-) diff --git a/backend/src/api/endpoints/entity.py b/backend/src/api/endpoints/entity.py index a2c8c3f7d..74ef28c3a 100644 --- a/backend/src/api/endpoints/entity.py +++ b/backend/src/api/endpoints/entity.py @@ -1,4 +1,3 @@ -from itertools import chain from typing import List from fastapi import APIRouter, Depends @@ -8,9 +7,7 @@ from app.core.authorization.authz_user import AuthzUser from app.core.data.crud import Crud from app.core.data.crud.entity import crud_entity -from app.core.data.crud.span_text import crud_span_text from app.core.data.dto.entity import ( - EntityCreate, EntityMerge, EntityRead, EntityRelease, @@ -53,23 +50,7 @@ def merge_entities( authz_user: AuthzUser = Depends(), ) -> EntityRead: authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_merge.entity_ids) - all_span_texts = ( - list( - chain.from_iterable( - [st.id for st in crud_entity.read(db=db, id=id).span_texts] - for id in entity_merge.entity_ids - ) - ) - + entity_merge.spantext_ids - ) - new_entity = EntityCreate( - name=entity_merge.name, - project_id=entity_merge.project_id, - span_text_ids=all_span_texts, - is_human=True, - knowledge_base_id=entity_merge.knowledge_base_id, - ) - db_obj = crud_entity.create(db=db, create_dto=new_entity, force=True) + db_obj = crud_entity.merge(db, entity_merge=entity_merge) return EntityRead.model_validate(db_obj) @@ -82,27 +63,9 @@ def merge_entities( def release_entities( *, db: Session = Depends(get_db_session), - entity_resolve: EntityRelease, + entity_release: EntityRelease, authz_user: AuthzUser = Depends(), -) -> EntityRead: - authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_resolve.entity_ids) - all_span_texts = ( - list( - chain.from_iterable( - [st.id for st in crud_entity.read(db=db, id=id).span_texts] - for id in entity_resolve.entity_ids - ) - ) - + entity_resolve.spantext_ids - ) - new_entities = [] - for span_text_id in all_span_texts: - span_text = crud_span_text.read(db=db, id=span_text_id) - new_entity = EntityCreate( - name=span_text.text, - project_id=entity_resolve.project_id, - span_text_ids=[span_text_id], - ) - new_entities.append(new_entity) - db_objs = crud_entity.create_multi(db=db, create_dtos=new_entities, force=True) +) -> List[EntityRead]: + authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_release.entity_ids) + db_objs = crud_entity.release(db=db, entity_release=entity_release) return [EntityRead.model_validate(db_obj) for db_obj in db_objs] diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py index 0946ff36e..5910dd303 100644 --- a/backend/src/app/core/data/crud/entity.py +++ b/backend/src/app/core/data/crud/entity.py @@ -1,13 +1,16 @@ -from typing import List, Optional +from itertools import chain +from typing import List from fastapi.encoders import jsonable_encoder -from sqlalchemy import select from sqlalchemy.orm import Session from app.core.data.crud.crud_base import CRUDBase +from app.core.data.crud.span_text import crud_span_text from app.core.data.crud.span_text_entity_link import crud_span_text_entity_link from app.core.data.dto.entity import ( EntityCreate, + EntityMerge, + EntityRelease, EntityUpdate, ) from app.core.data.dto.span_text_entity_link import ( @@ -29,6 +32,30 @@ def create_multi( ) -> List[EntityORM]: if len(create_dtos) == 0: return [] + + # assumption all entities belong to the same project + project_id = create_dtos[0].project_id + + # duplicate assignments to the same span text are filtered out here + span_text_dict = {} + for i, create_dto in enumerate(create_dtos): + for span_text_id in create_dto.span_text_ids: + span_text_dict[span_text_id] = i + + ids = list(span_text_dict.keys()) + existing_links = crud_span_text_entity_link.read_multi_span_text_and_project_id( + db=db, span_text_ids=ids, project_id=project_id + ) + existing_link_ids = [link.linked_span_text_id for link in existing_links] + old_entities = [link.linked_entity_id for link in existing_links] + + if not force: + # if a span text is already assigned it should not be reassigned + for id in existing_link_ids: + del span_text_dict[id] + + indexes_to_use = list(set(span_text_dict.values())) + create_dtos = [c for i, c in enumerate(create_dtos) if i in indexes_to_use] dto_objs_data = [ jsonable_encoder(dto, exclude={"span_text_ids"}) for dto in create_dtos ] @@ -45,9 +72,17 @@ def create_multi( linked_entity_id=db_obj.id, linked_span_text_id=span_text_id ) ) - crud_span_text_entity_link.create_multi(db=db, create_dtos=links, force=force) + crud_span_text_entity_link.create_multi(db=db, create_dtos=links) db.commit() - self.remove_all_unused_entites(db=db) + if force: + existing_links = ( + crud_span_text_entity_link.read_multi_span_text_and_project_id( + db=db, span_text_ids=ids, project_id=project_id + ) + ) + new_entities = [x.linked_entity_id for x in existing_links] + to_check = list(set(old_entities) - set(new_entities)) + self.remove_unused_entites(db=db, ids=to_check) return db_objs def read_by_project(self, db: Session, proj_id: int) -> List[EntityORM]: @@ -61,15 +96,58 @@ def remove_multi(self, db: Session, *, ids: List[int]) -> List[EntityORM]: db.commit() return removed - def remove_all_unused_entites(self, db: Session) -> List[EntityORM]: - subquery = select(SpanTextEntityLinkORM.linked_entity_id).distinct().subquery() - query = ( - db.query(EntityORM) - .outerjoin(subquery, EntityORM.id == subquery.c.linked_entity_id) - .filter(subquery.c.linked_entity_id.is_(None)) + def remove_unused_entites(self, db: Session, ids: List[int]) -> List[EntityORM]: + linked_ids_result = ( + db.query(SpanTextEntityLinkORM.linked_entity_id) + .filter(SpanTextEntityLinkORM.linked_entity_id.in_(ids)) + .distinct() + .all() ) - to_remove = query.all() - return self.remove_multi(db=db, ids=[e.id for e in to_remove]) + linked_ids = {item[0] for item in linked_ids_result} + ids = list(set(ids) - set(linked_ids)) + return self.remove_multi(db=db, ids=ids) + + def merge(self, db: Session, entity_merge: EntityMerge) -> EntityORM: + all_span_texts = ( + list( + chain.from_iterable( + [st.id for st in crud_entity.read(db=db, id=id).span_texts] + for id in entity_merge.entity_ids + ) + ) + + entity_merge.spantext_ids + ) + new_entity = EntityCreate( + name=entity_merge.name, + project_id=entity_merge.project_id, + span_text_ids=all_span_texts, + is_human=True, + knowledge_base_id=entity_merge.knowledge_base_id, + ) + return self.create(db=db, create_dto=new_entity, force=True) + + def release(self, db: Session, entity_release: EntityRelease) -> List[EntityORM]: + all_span_texts = ( + list( + chain.from_iterable( + [st.id for st in self.read(db=db, id=id).span_texts] + for id in entity_release.entity_ids + ) + ) + + entity_release.spantext_ids + ) + new_entities = [] + for span_text_id in all_span_texts: + span_text = crud_span_text.read(db=db, id=span_text_id) + new_entity = EntityCreate( + name=span_text.text, + project_id=entity_release.project_id, + span_text_ids=[span_text_id], + ) + new_entities.append(new_entity) + db_objs = self.create_multi(db=db, create_dtos=new_entities, force=True) + self.remove_unused_entites(db=db, ids=entity_release.entity_ids) + return db_objs crud_entity = CRUDEntity(EntityORM) diff --git a/backend/src/app/core/data/crud/span_annotation.py b/backend/src/app/core/data/crud/span_annotation.py index 6406ddcb2..8ec7ea944 100644 --- a/backend/src/app/core/data/crud/span_annotation.py +++ b/backend/src/app/core/data/crud/span_annotation.py @@ -131,7 +131,12 @@ def create_multi( + "\n" + str([type(id) for id in span_texts_orm]) + "\n" - + str([id.as_dict() for id in span_texts_orm]) + + str( + [ + id.as_dict() if not isinstance(id, tuple) else id + for id in span_texts_orm + ] + ) ) # create the SpanAnnotation (and link the SpanText via FK) diff --git a/backend/src/app/core/data/crud/span_text_entity_link.py b/backend/src/app/core/data/crud/span_text_entity_link.py index 687d3f8e3..f280b96cb 100644 --- a/backend/src/app/core/data/crud/span_text_entity_link.py +++ b/backend/src/app/core/data/crud/span_text_entity_link.py @@ -32,11 +32,11 @@ def create( return db_obj def create_multi( - self, db: Session, *, create_dtos: List[SpanTextEntityLinkCreate], force: bool + self, db: Session, *, create_dtos: List[SpanTextEntityLinkCreate] ) -> List[SpanTextEntityLinkORM]: - # One assumption is that all entities have the same project_id if len(create_dtos) == 0: return [] + # One assumption is that all entities have the same project_id project_id = ( db.query(EntityORM) .filter(EntityORM.id == create_dtos[0].linked_entity_id) @@ -60,24 +60,23 @@ def create_multi( db.bulk_save_objects(db_objs) db.commit() if len(existing_links) > 0: - if force: - existing_links_map = { - link.linked_span_text_id: link for link in existing_links - } - for create_dto in create_dtos: - if create_dto.linked_span_text_id in existing_links_map: - existing_links_map[ - create_dto.linked_span_text_id - ].linked_entity_id = create_dto.linked_entity_id - ids = [dto.id for dto in existing_links] - update_dtos = [ - SpanTextEntityLinkUpdate( - linked_entity_id=dto.linked_entity_id, - linked_span_text_id=dto.linked_span_text_id, - ) - for dto in existing_links - ] - self.update_multi(db, ids=ids, update_dtos=update_dtos) + existing_links_map = { + link.linked_span_text_id: link for link in existing_links + } + for create_dto in create_dtos: + if create_dto.linked_span_text_id in existing_links_map: + existing_links_map[ + create_dto.linked_span_text_id + ].linked_entity_id = create_dto.linked_entity_id + ids = [dto.id for dto in existing_links] + update_dtos = [ + SpanTextEntityLinkUpdate( + linked_entity_id=dto.linked_entity_id, + linked_span_text_id=dto.linked_span_text_id, + ) + for dto in existing_links + ] + self.update_multi(db, ids=ids, update_dtos=update_dtos) def update( self, db: Session, *, id: int, update_dto: SpanTextEntityLinkUpdate From 9fb795a2c19c68e4ba7e4a55b3a3dcbdb9041783 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 4 Jul 2024 14:54:41 +0000 Subject: [PATCH 11/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- frontend/src/api/EntityHooks.ts | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/frontend/src/api/EntityHooks.ts b/frontend/src/api/EntityHooks.ts index 243471faf..0ff97a23e 100644 --- a/frontend/src/api/EntityHooks.ts +++ b/frontend/src/api/EntityHooks.ts @@ -3,8 +3,6 @@ import queryClient from "../plugins/ReactQueryClient.ts"; import { QueryKey } from "./QueryKey.ts"; import { EntityService } from "./openapi/services/EntityService.ts"; - - const useUpdateEntity = () => useMutation({ mutationFn: EntityService.updateById, @@ -18,25 +16,24 @@ const useMerge = () => useMutation({ mutationFn: EntityService.mergeEntities, onSuccess: (data) => { - queryClient.invalidateQueries({queryKey: [QueryKey.ENTITY, data.project_id]}); - queryClient.invalidateQueries({queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id]}) - } - }) - - const useRelease = () => - useMutation({ - mutationFn: EntityService.releaseEntities, - onSuccess: (data) => { - queryClient.invalidateQueries({queryKey: [QueryKey.ENTITY, data[0].project_id]}); - queryClient.invalidateQueries({queryKey: [QueryKey.PROJECT_ENTITIES, data[0].project_id]}) - } - }) + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data.project_id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data.project_id] }); + }, + }); +const useRelease = () => + useMutation({ + mutationFn: EntityService.releaseEntities, + onSuccess: (data) => { + queryClient.invalidateQueries({ queryKey: [QueryKey.ENTITY, data[0].project_id] }); + queryClient.invalidateQueries({ queryKey: [QueryKey.PROJECT_ENTITIES, data[0].project_id] }); + }, + }); const EntityHooks = { useUpdateEntity, useMerge, - useRelease + useRelease, }; export default EntityHooks; From 26e9791f803b734fad8661106fe7da2011858bd8 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Tue, 6 Aug 2024 00:48:39 +0000 Subject: [PATCH 12/18] Activated SubRow select, and removed entity_ids from merge and release dtos --- backend/src/api/endpoints/entity.py | 4 +- backend/src/app/core/data/crud/entity.py | 52 +++++----------- backend/src/app/core/data/dto/entity.py | 10 +-- .../src/api/openapi/models/EntityMerge.ts | 4 -- .../src/api/openapi/models/EntityRelease.ts | 6 +- .../EntityDashboard/EntityDashboard.tsx | 16 ++--- .../analysis/EntityDashboard}/EntityTable.tsx | 61 +++++++++---------- 7 files changed, 60 insertions(+), 93 deletions(-) rename frontend/src/{components/entity => views/analysis/EntityDashboard}/EntityTable.tsx (78%) diff --git a/backend/src/api/endpoints/entity.py b/backend/src/api/endpoints/entity.py index 74ef28c3a..c452a0e8d 100644 --- a/backend/src/api/endpoints/entity.py +++ b/backend/src/api/endpoints/entity.py @@ -49,7 +49,7 @@ def merge_entities( entity_merge: EntityMerge, authz_user: AuthzUser = Depends(), ) -> EntityRead: - authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_merge.entity_ids) + authz_user.assert_in_project(entity_merge.project_id) db_obj = crud_entity.merge(db, entity_merge=entity_merge) return EntityRead.model_validate(db_obj) @@ -66,6 +66,6 @@ def release_entities( entity_release: EntityRelease, authz_user: AuthzUser = Depends(), ) -> List[EntityRead]: - authz_user.assert_in_same_project_as_many(Crud.ENTITY, entity_release.entity_ids) + authz_user.assert_in_project(entity_release.project_id) db_objs = crud_entity.release(db=db, entity_release=entity_release) return [EntityRead.model_validate(db_obj) for db_obj in db_objs] diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py index 5910dd303..058b4438e 100644 --- a/backend/src/app/core/data/crud/entity.py +++ b/backend/src/app/core/data/crud/entity.py @@ -55,11 +55,12 @@ def create_multi( del span_text_dict[id] indexes_to_use = list(set(span_text_dict.values())) - create_dtos = [c for i, c in enumerate(create_dtos) if i in indexes_to_use] - dto_objs_data = [ - jsonable_encoder(dto, exclude={"span_text_ids"}) for dto in create_dtos - ] - db_objs = [self.model(**data) for data in dto_objs_data] + db_objs = [] + # mit map lösen + for i in indexes_to_use: + create_dto = create_dtos[i] + dto_objs_data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) + db_objs.append(self.model(**dto_objs_data)) db.add_all(db_objs) db.flush() db.commit() @@ -75,20 +76,13 @@ def create_multi( crud_span_text_entity_link.create_multi(db=db, create_dtos=links) db.commit() if force: - existing_links = ( - crud_span_text_entity_link.read_multi_span_text_and_project_id( - db=db, span_text_ids=ids, project_id=project_id - ) - ) - new_entities = [x.linked_entity_id for x in existing_links] - to_check = list(set(old_entities) - set(new_entities)) - self.remove_unused_entites(db=db, ids=to_check) + self.__remove_unused_entites(db=db, ids=list(set(old_entities))) return db_objs def read_by_project(self, db: Session, proj_id: int) -> List[EntityORM]: return db.query(self.model).filter(self.model.project_id == proj_id).all() - def remove_multi(self, db: Session, *, ids: List[int]) -> List[EntityORM]: + def __remove_multi(self, db: Session, *, ids: List[int]) -> List[EntityORM]: removed = db.query(EntityORM).filter(EntityORM.id.in_(ids)).all() db.query(EntityORM).filter(EntityORM.id.in_(ids)).delete( synchronize_session=False @@ -96,7 +90,10 @@ def remove_multi(self, db: Session, *, ids: List[int]) -> List[EntityORM]: db.commit() return removed - def remove_unused_entites(self, db: Session, ids: List[int]) -> List[EntityORM]: + def remove(self, db: Session, *, id: int) -> EntityORM: + pass + + def __remove_unused_entites(self, db: Session, ids: List[int]) -> List[EntityORM]: linked_ids_result = ( db.query(SpanTextEntityLinkORM.linked_entity_id) .filter(SpanTextEntityLinkORM.linked_entity_id.in_(ids)) @@ -105,39 +102,21 @@ def remove_unused_entites(self, db: Session, ids: List[int]) -> List[EntityORM]: ) linked_ids = {item[0] for item in linked_ids_result} ids = list(set(ids) - set(linked_ids)) - return self.remove_multi(db=db, ids=ids) + return self.__remove_multi(db=db, ids=ids) def merge(self, db: Session, entity_merge: EntityMerge) -> EntityORM: - all_span_texts = ( - list( - chain.from_iterable( - [st.id for st in crud_entity.read(db=db, id=id).span_texts] - for id in entity_merge.entity_ids - ) - ) - + entity_merge.spantext_ids - ) new_entity = EntityCreate( name=entity_merge.name, project_id=entity_merge.project_id, - span_text_ids=all_span_texts, + span_text_ids=entity_merge.spantext_ids, is_human=True, knowledge_base_id=entity_merge.knowledge_base_id, ) return self.create(db=db, create_dto=new_entity, force=True) def release(self, db: Session, entity_release: EntityRelease) -> List[EntityORM]: - all_span_texts = ( - list( - chain.from_iterable( - [st.id for st in self.read(db=db, id=id).span_texts] - for id in entity_release.entity_ids - ) - ) - + entity_release.spantext_ids - ) new_entities = [] - for span_text_id in all_span_texts: + for span_text_id in entity_release.spantext_ids: span_text = crud_span_text.read(db=db, id=span_text_id) new_entity = EntityCreate( name=span_text.text, @@ -146,7 +125,6 @@ def release(self, db: Session, entity_release: EntityRelease) -> List[EntityORM] ) new_entities.append(new_entity) db_objs = self.create_multi(db=db, create_dtos=new_entities, force=True) - self.remove_unused_entites(db=db, ids=entity_release.entity_ids) return db_objs diff --git a/backend/src/app/core/data/dto/entity.py b/backend/src/app/core/data/dto/entity.py index 377e4fa72..68b2a55de 100644 --- a/backend/src/app/core/data/dto/entity.py +++ b/backend/src/app/core/data/dto/entity.py @@ -11,9 +11,9 @@ # Properties shared across all DTOs class EntityBaseDTO(BaseModel): is_human: Optional[bool] = Field( - False, description="Whether the link was created by a human" + default=False, description="Whether the link was created by a human" ) - knowledge_base_id: Optional[str] = Field("", description="Link to wikidata") + knowledge_base_id: Optional[str] = Field(default="", description="Link to wikidata") # Properties for creation @@ -35,19 +35,19 @@ class EntityUpdate(EntityBaseDTO, UpdateDTOBase): # Properties for merging entities/span texts +# TODO entity ids löschen und im frontend nur span_text ids weitergeben class EntityMerge(EntityBaseDTO): name: str = Field(description="Name of the Entity") knowledge_base_id: Optional[str] = Field("", description="Link to wikidata") project_id: int = Field(description="Id of the current Project") - entity_ids: List[int] = Field(description="List of Entity IDs to merge") spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") # Properties for releasing entities/span texts +# TODO entity ids löschen und im frontend nur span_text ids weitergeben class EntityRelease(EntityBaseDTO): project_id: int = Field(description="Id of the current Project") - entity_ids: List[int] = Field(description="List of Entity IDs to merge") - spantext_ids: List[int] = Field(description="List of Span Text IDs to merge") + spantext_ids: List[int] = Field(description="List of Span Text IDs to release") # Properties for reading (as in ORM) diff --git a/frontend/src/api/openapi/models/EntityMerge.ts b/frontend/src/api/openapi/models/EntityMerge.ts index e8ad9fd81..77241240a 100644 --- a/frontend/src/api/openapi/models/EntityMerge.ts +++ b/frontend/src/api/openapi/models/EntityMerge.ts @@ -19,10 +19,6 @@ export type EntityMerge = { * Id of the current Project */ project_id: number; - /** - * List of Entity IDs to merge - */ - entity_ids: Array; /** * List of Span Text IDs to merge */ diff --git a/frontend/src/api/openapi/models/EntityRelease.ts b/frontend/src/api/openapi/models/EntityRelease.ts index 9dbf3b8a7..c9f7ce659 100644 --- a/frontend/src/api/openapi/models/EntityRelease.ts +++ b/frontend/src/api/openapi/models/EntityRelease.ts @@ -16,11 +16,7 @@ export type EntityRelease = { */ project_id: number; /** - * List of Entity IDs to merge - */ - entity_ids: Array; - /** - * List of Span Text IDs to merge + * List of Span Text IDs to release */ spantext_ids: Array; }; diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx index 444ca195f..e0c8567dd 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -3,15 +3,14 @@ import { MRT_RowSelectionState, MRT_TableOptions } from "material-react-table"; import { useContext, useState } from "react"; import { useParams } from "react-router-dom"; import EntityHooks from "../../../api/EntityHooks.ts"; -import { EntityRead } from "../../../api/openapi/models/EntityRead.ts"; import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; +import { AppBarContext } from "../../../layouts/TwoBarLayout.tsx"; +import { useAppSelector } from "../../../plugins/ReduxHooks.ts"; import EntityTable, { EnitityTableRow, EntityTableSaveRowProps, SpanTextTableRow, -} from "../../../components/entity/EntityTable.tsx"; -import { AppBarContext } from "../../../layouts/TwoBarLayout.tsx"; -import { useAppSelector } from "../../../plugins/ReduxHooks.ts"; +} from "./EntityTable.tsx"; function EntityDashboard() { const appBarContainerRef = useContext(AppBarContext); @@ -26,11 +25,10 @@ function EntityDashboard() { const entityRelease = EntityHooks.useRelease(); const entityUpdate = EntityHooks.useUpdateEntity(); - function handleRelease(selectedEntities: EntityRead[], selectedSpanTexts: SpanTextRead[]): void { + function handleRelease(selectedSpanTexts: SpanTextRead[]): void { const requestBody = { requestBody: { project_id: projectId, - entity_ids: selectedEntities.map((entity) => entity.id), spantext_ids: selectedSpanTexts.map((spantext) => spantext.id), }, }; @@ -57,13 +55,13 @@ function EntityDashboard() { function handleMerge(props: EntityTableSaveRowProps): void { props.table.setCreatingRow(null); + console.log(props.selectedSpanTexts) const name = props.values.name; const knowledge_base_id = props.values.knowledge_base_id; const requestBody = { requestBody: { name: name, project_id: projectId, - entity_ids: props.selectedEntities.map((entity) => entity.id), spantext_ids: props.selectedSpanTexts.map((spantext) => spantext.id), knowledge_base_id: knowledge_base_id, }, @@ -72,6 +70,8 @@ function EntityDashboard() { setRowSelectionModel({}); } + + return ( @@ -90,7 +90,7 @@ function EntityDashboard() { - diff --git a/frontend/src/components/entity/EntityTable.tsx b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx similarity index 78% rename from frontend/src/components/entity/EntityTable.tsx rename to frontend/src/views/analysis/EntityDashboard/EntityTable.tsx index 3a09ddf1e..f1add9504 100644 --- a/frontend/src/components/entity/EntityTable.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx @@ -8,9 +8,9 @@ import { useMaterialReactTable, } from "material-react-table"; import { useMemo } from "react"; -import ProjectHooks from "../../api/ProjectHooks.ts"; -import { EntityRead } from "../../api/openapi/models/EntityRead.ts"; -import { SpanTextRead } from "../../api/openapi/models/SpanTextRead.ts"; +import ProjectHooks from "../../../api/ProjectHooks.ts"; +import { EntityRead } from "../../../api/openapi/models/EntityRead.ts"; +import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; export interface EnitityTableRow extends EntityRead { table_id: string; @@ -45,7 +45,6 @@ const columns: MRT_ColumnDef[] = [ header: "Is Human", enableEditing: false, Cell: ({ cell }) => { - console.log(cell); return cell.getValue() ? "True" : "False"; }, }, @@ -53,7 +52,6 @@ const columns: MRT_ColumnDef[] = [ export interface EntityTableActionProps { table: MRT_TableInstance; - selectedEntities: EntityRead[]; selectedSpanTexts: SpanTextRead[]; } @@ -91,22 +89,24 @@ function EntityTable({ const projectEntities = ProjectHooks.useGetAllEntities(projectId); // computed - const { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap } = useMemo(() => { + const {projectEntitiesRows, projectSpanTextMap } = useMemo(() => { if (!projectEntities.data) + { return { projectEntitiesMap: {} as Record, projectEntitiesRows: [], projectSpanTextMap: {} as Record, }; + } - const projectEntitiesMap = projectEntities.data.reduce( - (entity_map, projectEntity) => { - const id = `E-${projectEntity.id}`; - entity_map[id] = projectEntity; - return entity_map; - }, - {} as Record, - ); + //const projectEntitiesMap = projectEntities.data.reduce( + // (entity_map, projectEntity) => { + // const id = `E-${projectEntity.id}`; + // entity_map[id] = projectEntity; + // return entity_map; + // }, + // {} as Record, + //); const projectEntitiesRows = projectEntities.data.map((entity) => { const subRows = entity.span_texts?.map((span) => ({ @@ -123,7 +123,8 @@ function EntityTable({ const projectSpanTextMap = projectEntities.data.reduce( (acc, entity) => { - if (Array.isArray(entity.span_texts)) { + if (Array.isArray(entity.span_texts)) + { entity.span_texts.forEach((span) => { acc[`S-${span.id}`] = span; }); @@ -133,7 +134,7 @@ function EntityTable({ {} as Record, ); - return { projectEntitiesMap, projectEntitiesRows, projectSpanTextMap }; + return {projectEntitiesRows, projectSpanTextMap }; }, [projectEntities.data]); // table @@ -148,13 +149,18 @@ function EntityTable({ editDisplayMode: "row", onEditingRowSave: onSaveEditRow, onCreatingRowSave: (props) => { + //const entitySpanTexts = Object.keys(props.table.getState().rowSelection) + //.filter((id) => id.startsWith("E-")) + //.flatMap((entityId) => projectEntitiesMap[entityId].span_texts || []); + + const selectedSpanTexts = Object.keys(props.table.getState().rowSelection) + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]); + + const allSpanTexts = [ ...selectedSpanTexts];//[...entitySpanTexts, ...selectedSpanTexts]; + onCreateSaveRow({ - selectedEntities: Object.keys(props.table.getState().rowSelection) - .filter((id) => id.startsWith("E-")) - .map((entityId) => projectEntitiesMap[entityId]), - selectedSpanTexts: Object.keys(props.table.getState().rowSelection) - .filter((id) => id.startsWith("S-")) - .map((spanTextId) => projectSpanTextMap[spanTextId]), + selectedSpanTexts: allSpanTexts, values: props.values, table: props.table, }); @@ -194,9 +200,6 @@ function EntityTable({ ? (props) => renderTopToolbarCustomActions({ table: props.table, - selectedEntities: Object.keys(rowSelectionModel) - .filter((id) => id.startsWith("E-")) - .map((entityId) => projectEntitiesMap[entityId]), selectedSpanTexts: Object.keys(rowSelectionModel) .filter((id) => id.startsWith("S-")) .map((spanTextId) => projectSpanTextMap[spanTextId]), @@ -206,9 +209,6 @@ function EntityTable({ ? (props) => renderToolbarInternalActions({ table: props.table, - selectedEntities: Object.keys(rowSelectionModel) - .filter((id) => id.startsWith("E-")) - .map((entityId) => projectEntitiesMap[entityId]), selectedSpanTexts: Object.keys(rowSelectionModel) .filter((id) => id.startsWith("S-")) .map((spanTextId) => projectSpanTextMap[spanTextId]), @@ -218,9 +218,6 @@ function EntityTable({ ? (props) => renderBottomToolbarCustomActions({ table: props.table, - selectedEntities: Object.keys(rowSelectionModel) - .filter((id) => id.startsWith("E-")) - .map((entityId) => projectEntitiesMap[entityId]), selectedSpanTexts: Object.keys(rowSelectionModel) .filter((id) => id.startsWith("S-")) .map((spanTextId) => projectSpanTextMap[spanTextId]), @@ -236,7 +233,7 @@ function EntityTable({ enableExpanding: true, getSubRows: (originalRow) => originalRow.subRows, filterFromLeafRows: true, //search for child rows and preserve parent rows - enableSubRowSelection: false, + enableSubRowSelection: true, }); return ; From 0eb20525953ac66e22bfee8ac977b4e3d4ee9a14 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 00:50:15 +0000 Subject: [PATCH 13/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- backend/src/app/core/data/crud/entity.py | 1 - .../EntityDashboard/EntityDashboard.tsx | 14 +++----------- .../analysis/EntityDashboard/EntityTable.tsx | 18 ++++++++---------- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py index 058b4438e..cd83865b2 100644 --- a/backend/src/app/core/data/crud/entity.py +++ b/backend/src/app/core/data/crud/entity.py @@ -1,4 +1,3 @@ -from itertools import chain from typing import List from fastapi.encoders import jsonable_encoder diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx index e0c8567dd..56ec21f9e 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -6,11 +6,7 @@ import EntityHooks from "../../../api/EntityHooks.ts"; import { SpanTextRead } from "../../../api/openapi/models/SpanTextRead.ts"; import { AppBarContext } from "../../../layouts/TwoBarLayout.tsx"; import { useAppSelector } from "../../../plugins/ReduxHooks.ts"; -import EntityTable, { - EnitityTableRow, - EntityTableSaveRowProps, - SpanTextTableRow, -} from "./EntityTable.tsx"; +import EntityTable, { EnitityTableRow, EntityTableSaveRowProps, SpanTextTableRow } from "./EntityTable.tsx"; function EntityDashboard() { const appBarContainerRef = useContext(AppBarContext); @@ -55,7 +51,7 @@ function EntityDashboard() { function handleMerge(props: EntityTableSaveRowProps): void { props.table.setCreatingRow(null); - console.log(props.selectedSpanTexts) + console.log(props.selectedSpanTexts); const name = props.values.name; const knowledge_base_id = props.values.knowledge_base_id; const requestBody = { @@ -70,8 +66,6 @@ function EntityDashboard() { setRowSelectionModel({}); } - - return ( @@ -90,9 +84,7 @@ function EntityDashboard() { - + ); }} diff --git a/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx index f1add9504..19f5e490a 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx @@ -89,9 +89,8 @@ function EntityTable({ const projectEntities = ProjectHooks.useGetAllEntities(projectId); // computed - const {projectEntitiesRows, projectSpanTextMap } = useMemo(() => { - if (!projectEntities.data) - { + const { projectEntitiesRows, projectSpanTextMap } = useMemo(() => { + if (!projectEntities.data) { return { projectEntitiesMap: {} as Record, projectEntitiesRows: [], @@ -123,8 +122,7 @@ function EntityTable({ const projectSpanTextMap = projectEntities.data.reduce( (acc, entity) => { - if (Array.isArray(entity.span_texts)) - { + if (Array.isArray(entity.span_texts)) { entity.span_texts.forEach((span) => { acc[`S-${span.id}`] = span; }); @@ -134,7 +132,7 @@ function EntityTable({ {} as Record, ); - return {projectEntitiesRows, projectSpanTextMap }; + return { projectEntitiesRows, projectSpanTextMap }; }, [projectEntities.data]); // table @@ -152,12 +150,12 @@ function EntityTable({ //const entitySpanTexts = Object.keys(props.table.getState().rowSelection) //.filter((id) => id.startsWith("E-")) //.flatMap((entityId) => projectEntitiesMap[entityId].span_texts || []); - + const selectedSpanTexts = Object.keys(props.table.getState().rowSelection) - .filter((id) => id.startsWith("S-")) - .map((spanTextId) => projectSpanTextMap[spanTextId]); + .filter((id) => id.startsWith("S-")) + .map((spanTextId) => projectSpanTextMap[spanTextId]); - const allSpanTexts = [ ...selectedSpanTexts];//[...entitySpanTexts, ...selectedSpanTexts]; + const allSpanTexts = [...selectedSpanTexts]; //[...entitySpanTexts, ...selectedSpanTexts]; onCreateSaveRow({ selectedSpanTexts: allSpanTexts, From 7040ba102f4c528de41da2c70ce11d8f9f43f4f7 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Tue, 6 Aug 2024 01:31:27 +0000 Subject: [PATCH 14/18] Added Typing projectEntitiesRows --- .../analysis/EntityDashboard/EntityTable.tsx | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx index f1add9504..c3634bd35 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx @@ -99,16 +99,8 @@ function EntityTable({ }; } - //const projectEntitiesMap = projectEntities.data.reduce( - // (entity_map, projectEntity) => { - // const id = `E-${projectEntity.id}`; - // entity_map[id] = projectEntity; - // return entity_map; - // }, - // {} as Record, - //); - const projectEntitiesRows = projectEntities.data.map((entity) => { - const subRows = + const projectEntitiesRows: EnitityTableRow[] = projectEntities.data.map((entity) => { + const subRows: SpanTextTableRow[] = entity.span_texts?.map((span) => ({ ...span, table_id: `S-${span.id}`, @@ -149,15 +141,11 @@ function EntityTable({ editDisplayMode: "row", onEditingRowSave: onSaveEditRow, onCreatingRowSave: (props) => { - //const entitySpanTexts = Object.keys(props.table.getState().rowSelection) - //.filter((id) => id.startsWith("E-")) - //.flatMap((entityId) => projectEntitiesMap[entityId].span_texts || []); - const selectedSpanTexts = Object.keys(props.table.getState().rowSelection) .filter((id) => id.startsWith("S-")) .map((spanTextId) => projectSpanTextMap[spanTextId]); - const allSpanTexts = [ ...selectedSpanTexts];//[...entitySpanTexts, ...selectedSpanTexts]; + const allSpanTexts = [ ...selectedSpanTexts]; onCreateSaveRow({ selectedSpanTexts: allSpanTexts, From 030d92afb8b9585ef78f45656df4ecd004d422f8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 01:36:05 +0000 Subject: [PATCH 15/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- frontend/src/views/analysis/EntityDashboard/EntityTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx index 83e6d4260..c869684c5 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityTable.tsx @@ -143,7 +143,7 @@ function EntityTable({ .filter((id) => id.startsWith("S-")) .map((spanTextId) => projectSpanTextMap[spanTextId]); - const allSpanTexts = [ ...selectedSpanTexts]; + const allSpanTexts = [...selectedSpanTexts]; onCreateSaveRow({ selectedSpanTexts: allSpanTexts, From 7f7ab0a0e5a2734c77d8cba0af04e805bbcacff7 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Tue, 6 Aug 2024 01:38:19 +0000 Subject: [PATCH 16/18] Missed files/console.log --- frontend/src/openapi.json | 18 +++--------------- .../EntityDashboard/EntityDashboard.tsx | 1 - 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/frontend/src/openapi.json b/frontend/src/openapi.json index b04831d6f..20dead3ff 100644 --- a/frontend/src/openapi.json +++ b/frontend/src/openapi.json @@ -6934,12 +6934,6 @@ }, "name": { "type": "string", "title": "Name", "description": "Name of the Entity" }, "project_id": { "type": "integer", "title": "Project Id", "description": "Id of the current Project" }, - "entity_ids": { - "items": { "type": "integer" }, - "type": "array", - "title": "Entity Ids", - "description": "List of Entity IDs to merge" - }, "spantext_ids": { "items": { "type": "integer" }, "type": "array", @@ -6948,7 +6942,7 @@ } }, "type": "object", - "required": ["name", "project_id", "entity_ids", "spantext_ids"], + "required": ["name", "project_id", "spantext_ids"], "title": "EntityMerge" }, "EntityRead": { @@ -7007,21 +7001,15 @@ "default": "" }, "project_id": { "type": "integer", "title": "Project Id", "description": "Id of the current Project" }, - "entity_ids": { - "items": { "type": "integer" }, - "type": "array", - "title": "Entity Ids", - "description": "List of Entity IDs to merge" - }, "spantext_ids": { "items": { "type": "integer" }, "type": "array", "title": "Spantext Ids", - "description": "List of Span Text IDs to merge" + "description": "List of Span Text IDs to release" } }, "type": "object", - "required": ["project_id", "entity_ids", "spantext_ids"], + "required": ["project_id", "spantext_ids"], "title": "EntityRelease" }, "EntityUpdate": { diff --git a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx index 56ec21f9e..ac36cbec6 100644 --- a/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx +++ b/frontend/src/views/analysis/EntityDashboard/EntityDashboard.tsx @@ -51,7 +51,6 @@ function EntityDashboard() { function handleMerge(props: EntityTableSaveRowProps): void { props.table.setCreatingRow(null); - console.log(props.selectedSpanTexts); const name = props.values.name; const knowledge_base_id = props.values.knowledge_base_id; const requestBody = { From 434402419e8184df6204dba4efa3171031665668 Mon Sep 17 00:00:00 2001 From: 2fittsch Date: Thu, 8 Aug 2024 22:10:13 +0000 Subject: [PATCH 17/18] Changes: fixed error in entity multi create (db objs and dtos were not aligned), changed spantexts to do as TODO intended, made it so that read_by_text returns the SpanTextORM as intended. --- backend/src/app/core/data/crud/entity.py | 26 ++++++++--- .../src/app/core/data/crud/span_annotation.py | 38 +++++----------- backend/src/app/core/data/crud/span_text.py | 43 ++++++++----------- 3 files changed, 50 insertions(+), 57 deletions(-) diff --git a/backend/src/app/core/data/crud/entity.py b/backend/src/app/core/data/crud/entity.py index cd83865b2..02b796b59 100644 --- a/backend/src/app/core/data/crud/entity.py +++ b/backend/src/app/core/data/crud/entity.py @@ -53,13 +53,27 @@ def create_multi( for id in existing_link_ids: del span_text_dict[id] + # recompute create dtos indexes_to_use = list(set(span_text_dict.values())) - db_objs = [] - # mit map lösen - for i in indexes_to_use: - create_dto = create_dtos[i] - dto_objs_data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) - db_objs.append(self.model(**dto_objs_data)) + reversed_span_text_dict = {} + for key, value in span_text_dict.items(): + if value not in reversed_span_text_dict: + reversed_span_text_dict[value] = [] + reversed_span_text_dict[value].append(key) + + def map_index_to_new_dto(index): + create_dto = create_dtos[index] + create_dto.span_text_ids = reversed_span_text_dict[index] + return create_dto + + create_dtos = list(map(map_index_to_new_dto, indexes_to_use)) + + # create entity db_objs + def create_db_obj(create_dto): + data = jsonable_encoder(create_dto, exclude={"span_text_ids"}) + return self.model(**data) + + db_objs = list(map(create_db_obj, create_dtos)) db.add_all(db_objs) db.flush() db.commit() diff --git a/backend/src/app/core/data/crud/span_annotation.py b/backend/src/app/core/data/crud/span_annotation.py index 8ec7ea944..bb44cfb88 100644 --- a/backend/src/app/core/data/crud/span_annotation.py +++ b/backend/src/app/core/data/crud/span_annotation.py @@ -110,34 +110,18 @@ def create_multi( # create the entities code = crud_code.read(db=db, id=create_dtos[0].current_code_id) project_id = code.project_id - try: - crud_entity.create_multi( - db=db, - create_dtos=[ - EntityCreate( - project_id=project_id, - name=dto.span_text, - span_text_ids=[id.id], - is_human=False, - ) - for id, dto in zip(span_texts_orm, create_dtos) - ], - ) - except Exception as e: - raise Exception( - str(e) - + "\n" - + str(span_texts_orm) - + "\n" - + str([type(id) for id in span_texts_orm]) - + "\n" - + str( - [ - id.as_dict() if not isinstance(id, tuple) else id - for id in span_texts_orm - ] + crud_entity.create_multi( + db=db, + create_dtos=[ + EntityCreate( + project_id=project_id, + name=dto.span_text, + span_text_ids=[id.id], + is_human=False, ) - ) + for id, dto in zip(span_texts_orm, create_dtos) + ], + ) # create the SpanAnnotation (and link the SpanText via FK) dto_objs_data = [ diff --git a/backend/src/app/core/data/crud/span_text.py b/backend/src/app/core/data/crud/span_text.py index 16c687c83..1993b2801 100644 --- a/backend/src/app/core/data/crud/span_text.py +++ b/backend/src/app/core/data/crud/span_text.py @@ -22,36 +22,31 @@ def create(self, db: Session, *, create_dto: SpanTextCreate) -> SpanTextORM: def create_multi( self, db: Session, *, create_dtos: List[SpanTextCreate] ) -> List[SpanTextORM]: - # Only create when not already present span_texts: List[SpanTextORM] = [] to_create: List[SpanTextCreate] = [] - span_text_idx: List[int] = [] - to_create_idx: List[int] = [] - text_create_map: Dict[str, int] = {} - - # TODO best would be "insert all (ignore existing) followed by get all" - for i, create_dto in enumerate(create_dtos): - db_obj = self.read_by_text(db=db, text=create_dto.text) - span_texts.append(db_obj) - if db_obj is None: - if create_dto.text not in text_create_map: - text_create_map[create_dto.text] = len(to_create) - to_create.append(create_dto) - span_text_idx.append(i) - to_create_idx.append(text_create_map[create_dto.text]) - if len(to_create) > 0: - created = super().create_multi(db=db, create_dtos=to_create) - for obj_idx, pos_idx in zip(to_create_idx, span_text_idx): - span_texts[pos_idx] = created[obj_idx] - # Ignore types: We've made sure that no `None` values remain since we've created - # span texts to replace them - return span_texts # type: ignore + + # every span text needs to be created at most once + span_text_dict = {} + for create_dto in create_dtos: + span_text_dict[create_dto.text] = create_dto + + # only create span texts when not already present + to_create = filter( + lambda x: self.read_by_text(db=db, text=x.text) is None, + span_text_dict.values(), + ) + super().create_multi(db=db, create_dtos=to_create) + + span_texts = list( + map(lambda x: self.read_by_text(db=db, text=x.text), create_dtos) + ) + return span_texts def read_by_text(self, db: Session, *, text: str) -> Optional[SpanTextORM]: - return db.query(self.model.id).filter(self.model.text == text).first() + return db.query(self.model).filter(self.model.text == text).first() def read_all_by_text(self, db: Session, *, texts: List[str]) -> List[SpanTextORM]: - return db.query(self.model.id).filter(self.model.text in texts) + return db.query(self.model).filter(self.model.text in texts) crud_span_text = CRUDSpanText(SpanTextORM) From ca824be719074167f70d3fc011b9f72785bc4314 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 22:11:19 +0000 Subject: [PATCH 18/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- backend/src/app/core/data/crud/span_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/app/core/data/crud/span_text.py b/backend/src/app/core/data/crud/span_text.py index 1993b2801..7da30af71 100644 --- a/backend/src/app/core/data/crud/span_text.py +++ b/backend/src/app/core/data/crud/span_text.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional +from typing import List, Optional from sqlalchemy.orm import Session