From ec0ac3c3bf7e7967520cb92b94229fd71267eacf Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 29 Aug 2024 11:44:50 +0200 Subject: [PATCH 01/14] feat: use new rest api client and prepare for support of separate document clients --- capella2polarion/__main__.py | 2 +- .../connectors/polarion_worker.py | 110 +++++++++++----- tests/conftest.py | 12 +- tests/test_cli.py | 24 ++-- tests/test_elements.py | 121 ++++++++++-------- tests/test_workitem_attachments.py | 116 +++++++++-------- 6 files changed, 235 insertions(+), 150 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 77a136d7..e0d3add7 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -173,7 +173,7 @@ def render_documents( polarion_worker.post_documents(new_documents) polarion_worker.update_documents(updated_documents) - polarion_worker.update_work_items(work_items) + polarion_worker.update_headings(work_items) if __name__ == "__main__": diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index b60ae11d..bced7057 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -68,19 +68,43 @@ def __init__( "Polarion PAT (Personal Access Token) parameter " "is not a set properly." ) - self.client = polarion_api.OpenAPIPolarionProjectClient( - self.polarion_params.project_id, - self.polarion_params.delete_work_items, + + self.polarion_client = polarion_api.PolarionClient( polarion_api_endpoint=f"{self.polarion_params.url}/rest/v1", polarion_access_token=self.polarion_params.private_access_token, - custom_work_item=data_models.CapellaWorkItem, + ) + self.project_client = self.polarion_client.generate_project_client( + project_id=self.polarion_params.project_id, + delete_status=( + "deleted" if self.polarion_params.delete_work_items else None + ), add_work_item_checksum=True, ) + self._additional_clients: dict[str, polarion_api.ProjectClient] = {} self.check_client() + def _get_client( + self, project_id: str | None + ) -> polarion_api.ProjectClient: + if project_id is None: + return self.project_client + if project_id in self._additional_clients: + return self._additional_clients[project_id] + client = self.polarion_client.generate_project_client( + project_id=project_id, + delete_status=( + "deleted" if self.polarion_params.delete_work_items else None + ), + add_work_item_checksum=True, + ) + if not client.exists(): + raise KeyError(f"Miss Polarion project with id {project_id}") + self._additional_clients[project_id] = client + return client + def check_client(self) -> None: """Instantiate the polarion client as member.""" - if not self.client.project_exists(): + if not self.project_client.exists(): raise KeyError( "Miss Polarion project with id " f"{self.polarion_params.project_id}" @@ -88,7 +112,7 @@ def check_client(self) -> None: def load_polarion_work_item_map(self): """Return a map from Capella UUIDs to Polarion work items.""" - work_items = self.client.get_all_work_items( + work_items = self.project_client.work_items.get_all( "HAS_VALUE:uuid_capella", {"workitems": "id,uuid_capella,checksum,status,type"}, ) @@ -117,7 +141,7 @@ def serialize_for_delete(uuid: str) -> str: work_item_ids = [serialize_for_delete(uuid) for uuid in uuids] if work_item_ids: try: - self.client.delete_work_items(work_item_ids) + self.project_client.work_items.delete(work_item_ids) self.polarion_data_repo.remove_work_items_by_capella_uuid( uuids ) @@ -146,7 +170,7 @@ def create_missing_work_items( logger.info("Create work item for %r...", work_item.title) if missing_work_items: try: - self.client.create_work_items(missing_work_items) + self.project_client.work_items.create(missing_work_items) self.polarion_data_repo.update_work_items(missing_work_items) except polarion_api.PolarionApiException as error: logger.error("Creating work items failed. %s", error.args[0]) @@ -184,18 +208,20 @@ def compare_and_update_work_item( work_item_changed = new_work_item_check_sum != old_work_item_check_sum try: if work_item_changed or self.force_update: - old = self.client.get_work_item(old.id) + old = self.project_client.work_items.get(old.id) if old.attachments: old_attachments = ( - self.client.get_all_work_item_attachments( + self.project_client.work_items.attachments.get_all( work_item_id=old.id ) ) else: old_attachments = [] else: - old_attachments = self.client.get_all_work_item_attachments( - work_item_id=old.id + old_attachments = ( + self.project_client.work_items.attachments.get_all( + work_item_id=old.id + ) ) if old_attachments or new.attachments: work_item_changed |= self.update_attachments( @@ -221,8 +247,8 @@ def compare_and_update_work_item( del old.additional_attributes["uuid_capella"] if old.linked_work_items_truncated: - old.linked_work_items = self.client.get_all_work_item_links( - old.id + old.linked_work_items = ( + self.project_client.work_items.links.get_all(old.id) ) # Type will only be updated, if set and should be used carefully @@ -256,7 +282,7 @@ def compare_and_update_work_item( new.title = None try: - self.client.update_work_item(new) + self.project_client.work_items.update(new) if delete_links: id_list_str = ", ".join(delete_links.keys()) logger.info( @@ -265,7 +291,9 @@ def compare_and_update_work_item( new.type, new.title, ) - self.client.delete_work_item_links(list(delete_links.values())) + self.project_client.work_items.links.delete( + list(delete_links.values()) + ) if create_links: id_list_str = ", ".join(create_links.keys()) @@ -275,7 +303,9 @@ def compare_and_update_work_item( new.type, new.title, ) - self.client.create_work_item_links(list(create_links.values())) + self.project_client.work_items.links.create( + list(create_links.values()) + ) except polarion_api.PolarionApiException as error: logger.error( @@ -346,12 +376,12 @@ def update_attachments( attachment.file_name, attachment.id, ) - self.client.delete_work_item_attachment(attachment) + self.project_client.work_items.attachments.delete(attachment) old_attachment_file_names = set(old_attachment_dict) new_attachment_file_names = set(new_attachment_dict) for file_name in old_attachment_file_names - new_attachment_file_names: - self.client.delete_work_item_attachment( + self.project_client.work_items.attachments.delete( old_attachment_dict[file_name] ) @@ -361,7 +391,7 @@ def update_attachments( new_attachment_file_names - old_attachment_file_names, ) ): - self.client.create_work_item_attachments(new_attachments) + self.project_client.work_items.attachments.create(new_attachments) created = True attachments_for_update = {} @@ -386,7 +416,7 @@ def update_attachments( ): continue - self.client.update_work_item_attachment(attachment) + self.project_client.work_items.attachments.update(attachment) return created @staticmethod @@ -425,20 +455,31 @@ def compare_and_update_work_items( if uuid in self.polarion_data_repo and data.work_item is not None: self.compare_and_update_work_item(data) - def post_documents(self, documents: list[polarion_api.Document]): + def post_documents( + self, + documents: list[polarion_api.Document], + document_project: str | None = None, + ): """Create new documents.""" - self.client.project_client.documents.create(documents) + client = self._get_client(document_project) + client.documents.create(documents) - def update_documents(self, documents: list[polarion_api.Document]): + def update_documents( + self, + documents: list[polarion_api.Document], + document_project: str | None = None, + ): """Update existing documents.""" - self.client.project_client.documents.update(documents) + client = self._get_client(document_project) + client.documents.update(documents) def get_document( - self, space: str, name: str + self, space: str, name: str, document_project: str | None = None ) -> polarion_api.Document | None: """Get a document from polarion and return None if not found.""" + client = self._get_client(document_project) try: - return self.client.project_client.documents.get( + return client.documents.get( space, name, fields={"documents": "@all"} ) except polarion_api.PolarionApiBaseException as e: @@ -446,15 +487,22 @@ def get_document( return None raise e - def update_work_items(self, work_items: list[polarion_api.WorkItem]): + def update_headings( + self, + work_items: list[polarion_api.WorkItem], + document_project: str | None = None, + ): """Update the given workitems without any additional checks.""" - self.client.project_client.work_items.update(work_items) + client = self._get_client(document_project) + client.work_items.update(work_items) def load_polarion_documents( - self, document_paths: t.Iterable[tuple[str, str]] + self, + document_paths: t.Iterable[tuple[str, str]], + document_project: str | None = None, ) -> dict[tuple[str, str], polarion_api.Document | None]: """Load the given document references from Polarion.""" return { - (space, name): self.get_document(space, name) + (space, name): self.get_document(space, name, document_project) for space, name in document_paths } diff --git a/tests/conftest.py b/tests/conftest.py index a1310ecf..2d5afc09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -128,8 +128,10 @@ def base_object( ) c2p_cli.setup_logger() - mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) - monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) + mock_api_client = mock.MagicMock(spec=polarion_api.PolarionClient) + monkeypatch.setattr(polarion_api, "PolarionClient", mock_api_client) + mock_project_client = mock.MagicMock(spec=polarion_api.ProjectClient) + monkeypatch.setattr(polarion_api, "ProjectClient", mock_project_client) c2p_cli.config = mock.Mock(converter_config.ConverterConfig) fake = FakeModelObject("uuid1", name="Fake 1") @@ -173,8 +175,10 @@ def base_object( @pytest.fixture def empty_polarion_worker(monkeypatch: pytest.MonkeyPatch): - mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) - monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) + mock_api_client = mock.MagicMock(spec=polarion_api.PolarionClient) + monkeypatch.setattr(polarion_api, "PolarionClient", mock_api_client) + mock_project_client = mock.MagicMock(spec=polarion_api.ProjectClient) + monkeypatch.setattr(polarion_api, "ProjectClient", mock_project_client) polarion_params = polarion_worker.PolarionWorkerParams( project_id="project_id", url=TEST_HOST, diff --git a/tests/test_cli.py b/tests/test_cli.py index 6d7516f6..45439278 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,8 +23,10 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): - mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) - monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) + mock_api_client = mock.MagicMock(spec=polarion_api.PolarionClient) + monkeypatch.setattr(polarion_api, "PolarionClient", mock_api_client) + mock_project_client = mock.MagicMock(spec=polarion_api.ProjectClient) + monkeypatch.setattr(polarion_api, "ProjectClient", mock_project_client) mock_get_polarion_wi_map = mock.MagicMock() monkeypatch.setattr( polarion_worker.CapellaPolarionWorker, @@ -86,8 +88,10 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): def test_render_documents(monkeypatch: pytest.MonkeyPatch): - mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) - monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) + mock_api_client = mock.MagicMock(spec=polarion_api.PolarionClient) + monkeypatch.setattr(polarion_api, "PolarionClient", mock_api_client) + mock_project_client = mock.MagicMock(spec=polarion_api.ProjectClient) + monkeypatch.setattr(polarion_api, "ProjectClient", mock_project_client) mock_get_polarion_wi_map = mock.MagicMock() monkeypatch.setattr( polarion_worker.CapellaPolarionWorker, @@ -95,7 +99,7 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): mock_get_polarion_wi_map, ) mock_get_document = mock.MagicMock() - mock_get_document.side_effect = lambda folder, name: ( + mock_get_document.side_effect = lambda folder, name, project_id: ( polarion_api.Document( module_folder=folder, module_name=name, @@ -125,11 +129,11 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): "update_documents", mock_update_documents, ) - mock_update_work_items = mock.MagicMock() + mock_update_headings = mock.MagicMock() monkeypatch.setattr( polarion_worker.CapellaPolarionWorker, - "update_work_items", - mock_update_work_items, + "update_headings", + mock_update_headings, ) command: list[str] = [ @@ -156,5 +160,5 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): assert len(mock_post_documents.call_args.args[0]) == 1 assert mock_update_documents.call_count == 1 assert len(mock_update_documents.call_args.args[0]) == 1 - assert mock_update_work_items.call_count == 1 - assert len(mock_update_work_items.call_args.args[0]) == 1 + assert mock_update_headings.call_count == 1 + assert len(mock_update_headings.call_args.args[0]) == 1 diff --git a/tests/test_elements.py b/tests/test_elements.py index fc19f1c5..7a1fffd4 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -254,15 +254,6 @@ def test_create_diagrams(diagr_base_object: BaseObjectContainer): cls="diagram", ) - @staticmethod - def test_create_diagrams_filters_non_diagram_elements( - diagr_base_object: BaseObjectContainer, - ): - # This test does not make any sense, but it also didn't before - pw = diagr_base_object.pw - diagr_base_object.mc.generate_work_items(pw.polarion_data_repo) - assert pw.client.generate_work_items.call_count == 0 - @staticmethod def test_delete_diagrams(diagr_base_object: BaseObjectContainer): pw = diagr_base_object.pw @@ -270,10 +261,12 @@ def test_delete_diagrams(diagr_base_object: BaseObjectContainer): diagr_base_object.mc.generate_work_items(pw.polarion_data_repo) pw.create_missing_work_items(diagr_base_object.mc.converter_session) pw.delete_orphaned_work_items(diagr_base_object.mc.converter_session) - assert pw.client is not None - assert pw.client.delete_work_items.call_count == 1 - assert pw.client.delete_work_items.call_args[0][0] == ["Diag-1"] - assert pw.client.generate_work_items.call_count == 0 + assert pw.project_client is not None + assert pw.project_client.work_items.delete.call_count == 1 + assert pw.project_client.work_items.delete.call_args[0][0] == [ + "Diag-1" + ] + assert pw.project_client.work_items.create.call_count == 0 class TestModelElements: @@ -755,8 +748,8 @@ def test_update_work_items( polarion_api_get_all_work_items = mock.MagicMock() polarion_api_get_all_work_items.return_value = polarion_work_item_list monkeypatch.setattr( - base_object.pw.client, - "get_all_work_items", + base_object.pw.project_client.work_items, + "get_all", polarion_api_get_all_work_items, ) @@ -778,24 +771,35 @@ def test_update_work_items( get_work_item_mock = mock.MagicMock() get_work_item_mock.return_value = polarion_work_item_list[0] monkeypatch.setattr( - base_object.pw.client, - "get_work_item", + base_object.pw.project_client.work_items, + "get", get_work_item_mock, ) base_object.pw.compare_and_update_work_items( base_object.mc.converter_session ) - assert base_object.pw.client is not None - assert base_object.pw.client.get_all_work_item_links.call_count == 0 - assert base_object.pw.client.delete_work_item_links.call_count == 0 - assert base_object.pw.client.create_work_item_links.call_count == 0 - assert base_object.pw.client.update_work_item.call_count == 1 - assert base_object.pw.client.get_work_item.call_count == 1 assert ( - base_object.pw.client.get_all_work_item_attachments.call_count == 0 + base_object.pw.project_client.work_items.links.get_all.call_count + == 0 + ) + assert ( + base_object.pw.project_client.work_items.links.delete.call_count + == 0 + ) + assert ( + base_object.pw.project_client.work_items.links.create.call_count + == 0 ) - work_item = base_object.pw.client.update_work_item.call_args[0][0] + assert base_object.pw.project_client.work_items.update.call_count == 1 + assert base_object.pw.project_client.work_items.get.call_count == 1 + assert ( + base_object.pw.project_client.work_items.attachments.get_all.call_count + == 0 + ) + work_item = base_object.pw.project_client.work_items.update.call_args[ + 0 + ][0] assert isinstance(work_item, data_models.CapellaWorkItem) assert work_item.id == "Obj-1" assert work_item.title == "Fake 1" @@ -821,8 +825,8 @@ def test_update_deleted_work_item( polarion_api_get_all_work_items = mock.MagicMock() polarion_api_get_all_work_items.return_value = polarion_work_item_list monkeypatch.setattr( - base_object.pw.client, - "get_all_work_items", + base_object.pw.project_client.work_items, + "get_all", polarion_api_get_all_work_items, ) @@ -846,24 +850,26 @@ def test_update_deleted_work_item( get_work_item_mock = mock.MagicMock() get_work_item_mock.return_value = polarion_work_item_list[0] monkeypatch.setattr( - base_object.pw.client, - "get_work_item", + base_object.pw.project_client.work_items, + "get", get_work_item_mock, ) base_object.pw.delete_orphaned_work_items( base_object.mc.converter_session ) - assert base_object.pw.client.update_work_item.called is False + assert base_object.pw.project_client.work_items.update.called is False base_object.pw.create_missing_work_items( base_object.mc.converter_session ) - assert base_object.pw.client.create_work_items.called is False + assert base_object.pw.project_client.work_items.create.called is False base_object.pw.compare_and_update_work_items( base_object.mc.converter_session ) - work_item = base_object.pw.client.update_work_item.call_args[0][0] + work_item = base_object.pw.project_client.work_items.update.call_args[ + 0 + ][0] assert isinstance(work_item, data_models.CapellaWorkItem) assert work_item.status == "open" @@ -897,8 +903,7 @@ def test_update_work_items_filters_work_items_with_same_checksum( base_object.mc.converter_session ) - assert base_object.pw.client is not None - assert base_object.pw.client.update_work_item.call_count == 0 + assert base_object.pw.project_client.work_items.update.call_count == 0 @staticmethod def test_update_work_items_same_checksum_force( @@ -931,8 +936,7 @@ def test_update_work_items_same_checksum_force( base_object.mc.converter_session ) - assert base_object.pw.client is not None - assert base_object.pw.client.update_work_item.call_count == 1 + assert base_object.pw.project_client.work_items.update.call_count == 1 @staticmethod def test_update_links_with_no_elements(base_object: BaseObjectContainer): @@ -944,7 +948,10 @@ def test_update_links_with_no_elements(base_object: BaseObjectContainer): base_object.mc.converter_session ) - assert base_object.pw.client.get_all_work_item_links.call_count == 0 + assert ( + base_object.pw.project_client.work_items.links.get_all.call_count + == 0 + ) @staticmethod def test_update_links(base_object: BaseObjectContainer): @@ -980,8 +987,7 @@ def test_update_links(base_object: BaseObjectContainer): ) ) - assert base_object.pw.client is not None - base_object.pw.client.get_all_work_item_links.side_effect = ( + base_object.pw.project_client.work_items.links.get_all.side_effect = ( [link], [], ) @@ -1001,7 +1007,7 @@ def test_update_links(base_object: BaseObjectContainer): work_item_1.linked_work_items_truncated = True work_item_2.linked_work_items_truncated = True - base_object.pw.client.get_work_item.side_effect = ( + base_object.pw.project_client.work_items.get.side_effect = ( work_item_1, work_item_2, ) @@ -1009,19 +1015,31 @@ def test_update_links(base_object: BaseObjectContainer): base_object.pw.compare_and_update_work_items( base_object.mc.converter_session ) - assert base_object.pw.client is not None - links = base_object.pw.client.get_all_work_item_links.call_args_list - assert base_object.pw.client.get_all_work_item_links.call_count == 2 + links = ( + base_object.pw.project_client.work_items.links.get_all.call_args_list + ) + assert ( + base_object.pw.project_client.work_items.links.get_all.call_count + == 2 + ) assert [links[0][0][0], links[1][0][0]] == ["Obj-1", "Obj-2"] - new_links = base_object.pw.client.create_work_item_links.call_args[0][ - 0 - ] - assert base_object.pw.client.create_work_item_links.call_count == 1 + new_links = ( + base_object.pw.project_client.work_items.links.create.call_args[0][ + 0 + ] + ) + assert ( + base_object.pw.project_client.work_items.links.create.call_count + == 1 + ) assert new_links == [expected_new_link] - assert base_object.pw.client.delete_work_item_links.call_count == 1 - assert base_object.pw.client.delete_work_item_links.call_args[0][ + assert ( + base_object.pw.project_client.work_items.links.delete.call_count + == 1 + ) + assert base_object.pw.project_client.work_items.links.delete.call_args[ 0 - ] == [link] + ][0] == [link] @staticmethod def test_patch_work_item_grouped_links( @@ -1095,9 +1113,8 @@ def mock_back_link(converter_data, back_links): base_object.pw.compare_and_update_work_items( base_object.mc.converter_session ) - assert base_object.pw.client is not None update_work_item_calls = ( - base_object.pw.client.update_work_item.call_args_list + base_object.pw.project_client.work_items.update.call_args_list ) assert len(update_work_item_calls) == 3 mock_grouped_links_calls = mock_grouped_links.call_args_list diff --git a/tests/test_workitem_attachments.py b/tests/test_workitem_attachments.py index 6e51889e..d9a07977 100644 --- a/tests/test_workitem_attachments.py +++ b/tests/test_workitem_attachments.py @@ -57,8 +57,10 @@ @pytest.fixture def worker(monkeypatch: pytest.MonkeyPatch): - mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) - monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) + mock_api_client = mock.MagicMock(spec=polarion_api.PolarionClient) + monkeypatch.setattr(polarion_api, "PolarionClient", mock_api_client) + mock_project_client = mock.MagicMock(spec=polarion_api.ProjectClient) + monkeypatch.setattr(polarion_api, "ProjectClient", mock_project_client) return polarion_worker.CapellaPolarionWorker( polarion_worker.PolarionWorkerParams( "TEST", @@ -118,11 +120,13 @@ def test_diagram_attachments_new( [data_models.CapellaWorkItem(WORKITEM_ID, uuid_capella=TEST_DIAG_UUID)] ) - worker.client.get_work_item.return_value = data_models.CapellaWorkItem( - WORKITEM_ID, uuid_capella=TEST_DIAG_UUID + worker.project_client.work_items.get.return_value = ( + data_models.CapellaWorkItem(WORKITEM_ID, uuid_capella=TEST_DIAG_UUID) + ) + worker.project_client.work_items.attachments = mock.MagicMock() + worker.project_client.work_items.attachments.create.side_effect = ( + set_attachment_ids ) - worker.client.create_work_item_attachments = mock.MagicMock() - worker.client.create_work_item_attachments.side_effect = set_attachment_ids converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( "", @@ -136,15 +140,15 @@ def test_diagram_attachments_new( converter.converter_session[TEST_DIAG_UUID] ) - assert worker.client.update_work_item.call_count == 1 - assert worker.client.create_work_item_attachments.call_count == 1 - assert worker.client.get_all_work_item_attachments.call_count == 0 + assert worker.project_client.work_items.update.call_count == 1 + assert worker.project_client.work_items.attachments.create.call_count == 1 + assert worker.project_client.work_items.attachments.get_all.call_count == 0 created_attachments: list[polarion_api.WorkItemAttachment] = ( - worker.client.create_work_item_attachments.call_args.args[0] + worker.project_client.work_items.attachments.create.call_args.args[0] ) work_item: data_models.CapellaWorkItem = ( - worker.client.update_work_item.call_args.args[0] + worker.project_client.work_items.update.call_args.args[0] ) assert len(created_attachments) == 2 @@ -180,11 +184,15 @@ def test_new_diagram( ] ) - worker.client.get_work_item.return_value = data_models.CapellaWorkItem( - WORKITEM_ID, uuid_capella=TEST_DIAG_UUID, checksum=checksum + worker.project_client.work_items.get.return_value = ( + data_models.CapellaWorkItem( + WORKITEM_ID, uuid_capella=TEST_DIAG_UUID, checksum=checksum + ) + ) + worker.project_client.work_items.attachments.create = mock.MagicMock() + worker.project_client.work_items.attachments.create.side_effect = ( + set_attachment_ids ) - worker.client.create_work_item_attachments = mock.MagicMock() - worker.client.create_work_item_attachments.side_effect = set_attachment_ids converter.converter_session[TEST_DIAG_UUID] = data_session.ConverterData( "", @@ -198,9 +206,9 @@ def test_new_diagram( converter.converter_session[TEST_DIAG_UUID] ) - assert worker.client.update_work_item.call_count == 1 - assert worker.client.create_work_item_attachments.call_count == 1 - assert worker.client.update_work_item.call_args.args[ + assert worker.project_client.work_items.update.call_count == 1 + assert worker.project_client.work_items.attachments.create.call_count == 1 + assert worker.project_client.work_items.update.call_args.args[ 0 ].description == TEST_DIAG_DESCR.format( title="Diagram", @@ -233,14 +241,16 @@ def test_diagram_attachments_updated( ), ] - worker.client.get_work_item.return_value = data_models.CapellaWorkItem( - WORKITEM_ID, - uuid_capella=TEST_DIAG_UUID, - attachments=existing_attachments, + worker.project_client.work_items.get.return_value = ( + data_models.CapellaWorkItem( + WORKITEM_ID, + uuid_capella=TEST_DIAG_UUID, + attachments=existing_attachments, + ) ) - worker.client.get_all_work_item_attachments = mock.MagicMock() - worker.client.get_all_work_item_attachments.return_value = ( + worker.project_client.work_items.attachments.get_all = mock.MagicMock() + worker.project_client.work_items.attachments.get_all.return_value = ( existing_attachments ) @@ -256,13 +266,13 @@ def test_diagram_attachments_updated( converter.converter_session[TEST_DIAG_UUID] ) - assert worker.client.update_work_item.call_count == 1 - assert worker.client.create_work_item_attachments.call_count == 0 - assert worker.client.update_work_item_attachment.call_count == 2 - assert worker.client.get_all_work_item_attachments.call_count == 1 + assert worker.project_client.work_items.update.call_count == 1 + assert worker.project_client.work_items.attachments.create.call_count == 0 + assert worker.project_client.work_items.attachments.update.call_count == 2 + assert worker.project_client.work_items.attachments.get_all.call_count == 1 work_item: data_models.CapellaWorkItem = ( - worker.client.update_work_item.call_args.args[0] + worker.project_client.work_items.update.call_args.args[0] ) assert work_item.description == TEST_DIAG_DESCR.format( @@ -292,8 +302,8 @@ def test_diagram_attachments_unchanged_work_item_changed( ) ] ) - worker.client.get_all_work_item_attachments = mock.MagicMock() - worker.client.get_all_work_item_attachments.return_value = [ + worker.project_client.work_items.attachments.get_all = mock.MagicMock() + worker.project_client.work_items.attachments.get_all.return_value = [ polarion_api.WorkItemAttachment( WORKITEM_ID, "SVG-ATTACHMENT", @@ -320,12 +330,12 @@ def test_diagram_attachments_unchanged_work_item_changed( converter.converter_session[TEST_DIAG_UUID] ) - assert worker.client.update_work_item.call_count == 1 - assert worker.client.create_work_item_attachments.call_count == 0 - assert worker.client.update_work_item_attachment.call_count == 0 + assert worker.project_client.work_items.update.call_count == 1 + assert worker.project_client.work_items.attachments.create.call_count == 0 + assert worker.project_client.work_items.attachments.update.call_count == 0 work_item: data_models.CapellaWorkItem = ( - worker.client.update_work_item.call_args.args[0] + worker.project_client.work_items.update.call_args.args[0] ) assert work_item.description == TEST_DIAG_DESCR.format( @@ -363,10 +373,10 @@ def test_diagram_attachments_fully_unchanged( converter.converter_session[TEST_DIAG_UUID] ) - assert worker.client.update_work_item.call_count == 0 - assert worker.client.create_work_item_attachments.call_count == 0 - assert worker.client.update_work_item_attachment.call_count == 0 - assert worker.client.get_all_work_item_attachments.call_count == 0 + assert worker.project_client.work_items.update.call_count == 0 + assert worker.project_client.work_items.attachments.create.call_count == 0 + assert worker.project_client.work_items.attachments.update.call_count == 0 + assert worker.project_client.work_items.attachments.get_all.call_count == 0 def test_add_context_diagram( @@ -385,21 +395,23 @@ def test_add_context_diagram( model.by_uuid(uuid), ) - worker.client.create_work_item_attachments = mock.MagicMock() - worker.client.create_work_item_attachments.side_effect = set_attachment_ids + worker.project_client.work_items.attachments.create = mock.MagicMock() + worker.project_client.work_items.attachments.create.side_effect = ( + set_attachment_ids + ) converter.generate_work_items(worker.polarion_data_repo, False, True) worker.compare_and_update_work_item(converter.converter_session[uuid]) - assert worker.client.update_work_item.call_count == 1 - assert worker.client.create_work_item_attachments.call_count == 1 + assert worker.project_client.work_items.update.call_count == 1 + assert worker.project_client.work_items.attachments.create.call_count == 1 created_attachments: list[polarion_api.WorkItemAttachment] = ( - worker.client.create_work_item_attachments.call_args.args[0] + worker.project_client.work_items.attachments.create.call_args.args[0] ) work_item: data_models.CapellaWorkItem = ( - worker.client.update_work_item.call_args.args[0] + worker.project_client.work_items.update.call_args.args[0] ) assert len(created_attachments) == 2 @@ -439,8 +451,8 @@ def test_diagram_delete_attachments( ) ] ) - worker.client.get_all_work_item_attachments = mock.MagicMock() - worker.client.get_all_work_item_attachments.return_value = [ + worker.project_client.work_items.attachments.get_all = mock.MagicMock() + worker.project_client.work_items.attachments.get_all.return_value = [ polarion_api.WorkItemAttachment( WORKITEM_ID, "SVG-ATTACHMENT", @@ -473,13 +485,13 @@ def test_diagram_delete_attachments( converter.converter_session[TEST_DIAG_UUID] ) - assert worker.client.update_work_item.call_count == 1 - assert worker.client.create_work_item_attachments.call_count == 0 - assert worker.client.update_work_item_attachment.call_count == 0 - assert worker.client.delete_work_item_attachment.call_count == 2 + assert worker.project_client.work_items.update.call_count == 1 + assert worker.project_client.work_items.attachments.create.call_count == 0 + assert worker.project_client.work_items.attachments.update.call_count == 0 + assert worker.project_client.work_items.attachments.delete.call_count == 2 work_item: data_models.CapellaWorkItem = ( - worker.client.update_work_item.call_args.args[0] + worker.project_client.work_items.update.call_args.args[0] ) assert work_item.description is None From c5e096bd1b10a161e766e52306649444c9758ad6 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 29 Aug 2024 12:39:00 +0200 Subject: [PATCH 02/14] feat: add filtering for document statuses and support for document projects defined in the rendering config --- .../connectors/polarion_worker.py | 11 ++--- .../converters/document_config.py | 6 ++- .../converters/document_renderer.py | 40 ++++++++++++++++--- .../data/documents/full_authority_config.yaml | 4 ++ tests/data/documents/mixed_config.yaml | 4 ++ tests/test_documents.py | 16 ++++++-- 6 files changed, 66 insertions(+), 15 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index bced7057..c53e0db9 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -498,11 +498,12 @@ def update_headings( def load_polarion_documents( self, - document_paths: t.Iterable[tuple[str, str]], - document_project: str | None = None, - ) -> dict[tuple[str, str], polarion_api.Document | None]: + document_paths: t.Iterable[tuple[str | None, str, str]], + ) -> dict[tuple[str | None, str, str], polarion_api.Document | None]: """Load the given document references from Polarion.""" return { - (space, name): self.get_document(space, name, document_project) - for space, name in document_paths + (document_project, space, name): self.get_document( + space, name, document_project + ) + for document_project, space, name in document_paths } diff --git a/capella2polarion/converters/document_config.py b/capella2polarion/converters/document_config.py index 30ab8df6..ad039924 100644 --- a/capella2polarion/converters/document_config.py +++ b/capella2polarion/converters/document_config.py @@ -47,6 +47,8 @@ class BaseDocumentRenderingConfig(pydantic.BaseModel): """A template config, which can result in multiple Polarion documents.""" template_directory: str | pathlib.Path + project_id: str | None = None + status_allow_list: list[str] | None = None heading_numbering: bool = False work_item_layouts: dict[str, WorkItemLayout] = pydantic.Field( default_factory=dict @@ -77,11 +79,11 @@ class DocumentConfigs(pydantic.BaseModel): pydantic.Field(default_factory=list) ) - def iterate_documents(self) -> t.Iterator[tuple[str, str]]: + def iterate_documents(self) -> t.Iterator[tuple[str | None, str, str]]: """Yield all document paths of the config as tuples.""" for conf in self.full_authority + self.mixed_authority: for inst in conf.instances: - yield inst.polarion_space, inst.polarion_name + yield conf.project_id, inst.polarion_space, inst.polarion_name def read_config_file( diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index 7b012754..028785b0 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -292,16 +292,17 @@ def update_mixed_authority_document( def _get_and_customize_doc( self, + project_id: str | None, space: str, name: str, title: str | None, rendering_layouts: list[polarion_api.RenderingLayout], heading_numbering: bool, existing_documents: dict[ - tuple[str, str], polarion_api.Document | None + tuple[str | None, str, str], polarion_api.Document | None ], ) -> polarion_api.Document | None: - if old_doc := existing_documents.get((space, name)): + if old_doc := existing_documents.get((project_id, space, name)): if title: old_doc.title = title if self.overwrite_layouts: @@ -315,7 +316,7 @@ def render_documents( self, configs: document_config.DocumentConfigs, existing_documents: dict[ - tuple[str, str], polarion_api.Document | None + tuple[str | None, str, str], polarion_api.Document | None ], ) -> tuple[ list[polarion_api.Document], @@ -348,13 +349,33 @@ def render_documents( return new_docs, updated_docs, work_items + def _check_document_status( + self, + document: polarion_api.Document, + config: document_config.BaseDocumentRenderingConfig, + ): + if ( + config.status_allow_list is not None + and document.status not in config.status_allow_list + ): + logger.warning( + "Won't update document %s/%s due to status " + "restrictions. Status is %s and should be in %r.", + document.module_folder, + document.module_name, + document.status, + config.status_allow_list, + ) + return False + return True + def _render_mixed_authority_documents( self, mixed_authority_configs: list[ document_config.FullAuthorityDocumentRenderingConfig ], existing_documents: dict[ - tuple[str, str], polarion_api.Document | None + tuple[str | None, str, str], polarion_api.Document | None ], updated_docs: list[polarion_api.Document], work_items: list[polarion_api.WorkItem], @@ -365,6 +386,7 @@ def _render_mixed_authority_documents( ) for instance in config.instances: old_doc = self._get_and_customize_doc( + config.project_id, instance.polarion_space, instance.polarion_name, instance.polarion_title, @@ -380,6 +402,10 @@ def _render_mixed_authority_documents( instance.polarion_name, ) continue + + if not self._check_document_status(old_doc, config): + continue + try: new_doc, wis = self.update_mixed_authority_document( old_doc, @@ -405,7 +431,7 @@ def _render_full_authority_documents( self, full_authority_configs, existing_documents: dict[ - tuple[str, str], polarion_api.Document | None + tuple[str | None, str, str], polarion_api.Document | None ], new_docs: list[polarion_api.Document], updated_docs: list[polarion_api.Document], @@ -417,6 +443,7 @@ def _render_full_authority_documents( ) for instance in config.instances: if old_doc := self._get_and_customize_doc( + config.project_id, instance.polarion_space, instance.polarion_name, instance.polarion_title, @@ -424,6 +451,9 @@ def _render_full_authority_documents( config.heading_numbering, existing_documents, ): + if not self._check_document_status(old_doc, config): + continue + try: new_doc, wis = self.render_document( config.template_directory, diff --git a/tests/data/documents/full_authority_config.yaml b/tests/data/documents/full_authority_config.yaml index e56aca98..fdbb9899 100644 --- a/tests/data/documents/full_authority_config.yaml +++ b/tests/data/documents/full_authority_config.yaml @@ -3,6 +3,10 @@ - template_directory: jupyter-notebooks/document_templates template: test-icd.html.j2 + project_id: TestProject + status_allow_list: + - draft + - open instances: - polarion_space: _default polarion_name: id123 diff --git a/tests/data/documents/mixed_config.yaml b/tests/data/documents/mixed_config.yaml index a0d30924..fb346aa8 100644 --- a/tests/data/documents/mixed_config.yaml +++ b/tests/data/documents/mixed_config.yaml @@ -3,6 +3,10 @@ mixed_authority: - template_directory: jupyter-notebooks/document_templates + project_id: TestProject + status_allow_list: + - draft + - open sections: section1: test-icd.html.j2 section2: test-icd.html.j2 diff --git a/tests/test_documents.py b/tests/test_documents.py index 9c56332e..8768fe82 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -19,9 +19,11 @@ MIXED_AUTHORITY_DOCUMENT = TEST_DOCUMENT_ROOT / "mixed_authority_doc.html" -def existing_documents() -> dict[tuple[str, str], polarion_api.Document]: +def existing_documents() -> ( + dict[tuple[str | None, str, str], polarion_api.Document] +): return { - ("_default", "id123"): polarion_api.Document( + (None, "_default", "id123"): polarion_api.Document( module_folder="_default", module_name="id123", home_page_content=polarion_api.TextContent( @@ -34,7 +36,7 @@ def existing_documents() -> dict[tuple[str, str], polarion_api.Document]: ) ], ), - ("_default", "id1237"): polarion_api.Document( + (None, "_default", "id1237"): polarion_api.Document( module_folder="_default", module_name="id1237", home_page_content=polarion_api.TextContent( @@ -289,6 +291,10 @@ def test_full_authority_document_config(): assert conf.full_authority[0].instances[0].params == { "interface": "3d21ab4b-7bf6-428b-ba4c-a27bca4e86db" } + assert conf.full_authority[0].project_id == "TestProject" + assert conf.full_authority[0].status_allow_list == ["draft", "open"] + assert conf.full_authority[1].project_id is None + assert conf.full_authority[1].status_allow_list is None def test_mixed_authority_document_config(): @@ -308,6 +314,8 @@ def test_mixed_authority_document_config(): assert len(conf.mixed_authority[0].instances) == 2 assert conf.mixed_authority[0].instances[0].polarion_space == "_default" assert conf.mixed_authority[0].instances[0].polarion_name == "id123" + assert conf.mixed_authority[0].project_id == "TestProject" + assert conf.mixed_authority[0].status_allow_list == ["draft", "open"] assert conf.mixed_authority[0].instances[0].polarion_title == "Interface23" assert conf.mixed_authority[0].instances[0].params == { "interface": "3d21ab4b-7bf6-428b-ba4c-a27bca4e86db" @@ -315,6 +323,8 @@ def test_mixed_authority_document_config(): assert conf.mixed_authority[1].instances[0].section_params == { "section1": {"param_1": "Test"} } + assert conf.mixed_authority[1].project_id is None + assert conf.mixed_authority[1].status_allow_list is None def test_combined_config(): From cc95ad18dd7352d4102d41f271e8d20096d05f33 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 29 Aug 2024 18:01:01 +0200 Subject: [PATCH 03/14] feat: additional enhancements and proper testing for the changes --- capella2polarion/__main__.py | 12 ++-- .../converters/document_renderer.py | 56 +++++++++--------- tests/data/documents/combined_config.yaml | 26 +++++++++ tests/test_cli.py | 36 +++++++++--- tests/test_documents.py | 58 +++++++++++++------ 5 files changed, 131 insertions(+), 57 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index e0d3add7..728568b8 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -167,13 +167,11 @@ def render_documents( overwrite_layouts, ) - new_documents, updated_documents, work_items = renderer.render_documents( - configs, documents - ) - - polarion_worker.post_documents(new_documents) - polarion_worker.update_documents(updated_documents) - polarion_worker.update_headings(work_items) + projects_document_data = renderer.render_documents(configs, documents) + for project, project_data in projects_document_data.items(): + polarion_worker.post_documents(project_data.new_docs, project) + polarion_worker.update_documents(project_data.updated_docs, project) + polarion_worker.update_headings(project_data.work_items, project) if __name__ == "__main__": diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index 028785b0..31ae4bfc 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -37,6 +37,21 @@ class RenderingSession: ) +@dataclasses.dataclass +class ProjectData: + """A class holding data of a project which documents are rendered for.""" + + new_docs: list[polarion_api.Document] = dataclasses.field( + default_factory=list + ) + updated_docs: list[polarion_api.Document] = dataclasses.field( + default_factory=list + ) + work_items: list[polarion_api.WorkItem] = dataclasses.field( + default_factory=list + ) + + class DocumentRenderer(polarion_html_helper.JinjaRendererMixin): """A Renderer class for Polarion documents.""" @@ -52,6 +67,7 @@ def __init__( self.jinja_envs: dict[str, jinja2.Environment] = {} self.overwrite_heading_numbering = overwrite_heading_numbering self.overwrite_layouts = overwrite_layouts + self.projects: dict[str | None, ProjectData] = {} def setup_env(self, env: jinja2.Environment): """Add globals and filters to the environment.""" @@ -318,36 +334,23 @@ def render_documents( existing_documents: dict[ tuple[str | None, str, str], polarion_api.Document | None ], - ) -> tuple[ - list[polarion_api.Document], - list[polarion_api.Document], - list[polarion_api.WorkItem], - ]: + ) -> dict[str | None, ProjectData]: """Render all documents defined in the given config. Returns a list new documents followed by updated documents and work items, which need to be updated """ - new_docs: list[polarion_api.Document] = [] - updated_docs: list[polarion_api.Document] = [] - work_items: list[polarion_api.WorkItem] = [] self._render_full_authority_documents( configs.full_authority, existing_documents, - new_docs, - updated_docs, - work_items, ) self._render_mixed_authority_documents( - configs.mixed_authority, - existing_documents, - updated_docs, - work_items, + configs.mixed_authority, existing_documents ) - return new_docs, updated_docs, work_items + return self.projects def _check_document_status( self, @@ -377,13 +380,14 @@ def _render_mixed_authority_documents( existing_documents: dict[ tuple[str | None, str, str], polarion_api.Document | None ], - updated_docs: list[polarion_api.Document], - work_items: list[polarion_api.WorkItem], ): for config in mixed_authority_configs: rendering_layouts = document_config.generate_work_item_layouts( config.work_item_layouts ) + project_data = self.projects.setdefault( + config.project_id, ProjectData() + ) for instance in config.instances: old_doc = self._get_and_customize_doc( config.project_id, @@ -424,8 +428,8 @@ def _render_mixed_authority_documents( ) continue - updated_docs.append(new_doc) - work_items.extend(wis) + project_data.updated_docs.append(new_doc) + project_data.work_items.extend(wis) def _render_full_authority_documents( self, @@ -433,14 +437,14 @@ def _render_full_authority_documents( existing_documents: dict[ tuple[str | None, str, str], polarion_api.Document | None ], - new_docs: list[polarion_api.Document], - updated_docs: list[polarion_api.Document], - work_items: list[polarion_api.WorkItem], ): for config in full_authority_configs: rendering_layouts = document_config.generate_work_item_layouts( config.work_item_layouts ) + project_data = self.projects.setdefault( + config.project_id, ProjectData() + ) for instance in config.instances: if old_doc := self._get_and_customize_doc( config.project_id, @@ -471,8 +475,8 @@ def _render_full_authority_documents( ) continue - updated_docs.append(new_doc) - work_items.extend(wis) + project_data.updated_docs.append(new_doc) + project_data.work_items.extend(wis) else: try: new_doc, _ = self.render_document( @@ -495,7 +499,7 @@ def _render_full_authority_documents( ) continue - new_docs.append(new_doc) + project_data.new_docs.append(new_doc) def _extract_section_areas(self, html_elements: list[etree._Element]): section_areas = {} diff --git a/tests/data/documents/combined_config.yaml b/tests/data/documents/combined_config.yaml index 6a1034b3..3660a6d2 100644 --- a/tests/data/documents/combined_config.yaml +++ b/tests/data/documents/combined_config.yaml @@ -37,6 +37,21 @@ mixed_authority: section_params: section1: param_1: Test + - template_directory: jupyter-notebooks/document_templates + sections: + section1: test-icd.html.j2 + section2: test-icd.html.j2 + heading_numbering: True + project_id: TestProject + status_allow_list: + - draft + - open + instances: + - polarion_space: _default + polarion_name: id1239 + section_params: + section1: + param_1: Test full_authority: - template_directory: jupyter-notebooks/document_templates template: test-icd.html.j2 @@ -66,3 +81,14 @@ full_authority: instances: - polarion_space: _default polarion_name: id1238 + - template_directory: jupyter-notebooks/document_templates + template: test-icd.html.j2 + project_id: TestProject + status_allow_list: + - draft + - open + instances: + - polarion_space: _default + polarion_name: id1240 + params: + interface: 2681f26a-e492-4e5d-8b33-92fb00a48622 diff --git a/tests/test_cli.py b/tests/test_cli.py index 45439278..b384e75e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -155,10 +155,32 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): assert result.exit_code == 0 assert mock_get_polarion_wi_map.call_count == 1 - assert mock_get_document.call_count == 6 - assert mock_post_documents.call_count == 1 - assert len(mock_post_documents.call_args.args[0]) == 1 - assert mock_update_documents.call_count == 1 - assert len(mock_update_documents.call_args.args[0]) == 1 - assert mock_update_headings.call_count == 1 - assert len(mock_update_headings.call_args.args[0]) == 1 + assert mock_get_document.call_count == 8 + assert [call.args[2] for call in mock_get_document.call_args_list] == [ + None, + None, + None, + "TestProject", + None, + None, + None, + "TestProject", + ] + + assert mock_post_documents.call_count == 2 + assert len(mock_post_documents.call_args_list[0].args[0]) == 1 + assert len(mock_post_documents.call_args_list[1].args[0]) == 1 + assert mock_post_documents.call_args_list[0].args[1] is None + assert mock_post_documents.call_args_list[1].args[1] == "TestProject" + + assert mock_update_documents.call_count == 2 + assert len(mock_update_documents.call_args_list[0].args[0]) == 1 + assert len(mock_update_documents.call_args_list[1].args[0]) == 0 + assert mock_update_documents.call_args_list[0].args[1] is None + assert mock_update_documents.call_args_list[1].args[1] == "TestProject" + + assert mock_update_headings.call_count == 2 + assert len(mock_update_headings.call_args_list[0].args[0]) == 1 + assert len(mock_update_headings.call_args_list[1].args[0]) == 0 + assert mock_update_headings.call_args_list[0].args[1] is None + assert mock_update_headings.call_args_list[1].args[1] == "TestProject" diff --git a/tests/test_documents.py b/tests/test_documents.py index 8768fe82..4eacbd89 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -26,6 +26,7 @@ def existing_documents() -> ( (None, "_default", "id123"): polarion_api.Document( module_folder="_default", module_name="id123", + status="draft", home_page_content=polarion_api.TextContent( type="text/html", value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), @@ -39,6 +40,25 @@ def existing_documents() -> ( (None, "_default", "id1237"): polarion_api.Document( module_folder="_default", module_name="id1237", + status="draft", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), + ), + ("TestProject", "_default", "id1239"): polarion_api.Document( + module_folder="_default", + module_name="id1239", + status="in_review", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), + ), + ("TestProject", "_default", "id1240"): polarion_api.Document( + module_folder="_default", + module_name="id1240", + status="draft", home_page_content=polarion_api.TextContent( type="text/html", value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), @@ -233,24 +253,27 @@ def test_render_all_documents_partially_successfully( empty_polarion_worker.polarion_data_repo, model ) - new_docs, updated_docs, work_items = renderer.render_documents( - conf, existing_documents() - ) + projects_data = renderer.render_documents(conf, existing_documents()) - # There are 6 documents in the config, we expect 3 rendering to fail - assert len(caplog.records) == 3 + # There are 8 documents in the config, we expect 4 rendering to fail + assert len(caplog.records) == 4 + # The first tree documents weren't rendered due to an error, the fourth + # wasn't rendered because of status restrictions, which is a just warning + assert [lr.levelno for lr in caplog.records] == [40, 40, 40, 30] # For one valid config we did not pass a document, so we expect a new one - assert len(new_docs) == 1 - # And two updated documents - assert len(updated_docs) == 2 - # In both existing documents we had 2 headings. In full authority mode + assert len(projects_data[None].new_docs) == 1 + # And three updated documents + assert len(projects_data[None].updated_docs) == 2 + assert len(projects_data["TestProject"].updated_docs) == 1 + # In all existing documents we had 2 headings. In full authority mode # both should be updated and in mixed authority mode only one of them as # the other is outside the rendering area - assert len(work_items) == 3 - assert len(updated_docs[0].rendering_layouts) == 0 - assert len(updated_docs[1].rendering_layouts) == 1 - assert updated_docs[0].outline_numbering is None - assert updated_docs[1].outline_numbering is None + assert len(projects_data[None].work_items) == 3 + assert len(projects_data["TestProject"].work_items) == 2 + assert len(projects_data[None].updated_docs[0].rendering_layouts) == 0 + assert len(projects_data[None].updated_docs[1].rendering_layouts) == 1 + assert projects_data[None].updated_docs[0].outline_numbering is None + assert projects_data[None].updated_docs[1].outline_numbering is None def test_render_all_documents_overwrite_headings_layouts( @@ -264,7 +287,8 @@ def test_render_all_documents_overwrite_headings_layouts( empty_polarion_worker.polarion_data_repo, model, True, True ) - _, updated_docs, _ = renderer.render_documents(conf, existing_documents()) + projects_data = renderer.render_documents(conf, existing_documents()) + updated_docs = projects_data[None].updated_docs assert len(updated_docs[0].rendering_layouts) == 2 assert len(updated_docs[1].rendering_layouts) == 2 @@ -331,8 +355,8 @@ def test_combined_config(): with open(TEST_COMBINED_DOCUMENT_CONFIG, "r", encoding="utf-8") as f: conf = document_config.read_config_file(f) - assert len(conf.full_authority) == 2 - assert len(conf.mixed_authority) == 2 + assert len(conf.full_authority) == 3 + assert len(conf.mixed_authority) == 3 def test_rendering_config(): From 1e6973bf5fafb763ae4923e9e2c098a8fc4b4c3a Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 30 Aug 2024 08:59:31 +0200 Subject: [PATCH 04/14] fix: fixes from smoke testing --- capella2polarion/connectors/polarion_worker.py | 2 +- capella2polarion/converters/polarion_html_helper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index c53e0db9..03dfd012 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -114,7 +114,7 @@ def load_polarion_work_item_map(self): """Return a map from Capella UUIDs to Polarion work items.""" work_items = self.project_client.work_items.get_all( "HAS_VALUE:uuid_capella", - {"workitems": "id,uuid_capella,checksum,status,type"}, + fields={"workitems": "id,uuid_capella,checksum,status,type"}, ) self.polarion_data_repo.update_work_items(work_items) diff --git a/capella2polarion/converters/polarion_html_helper.py b/capella2polarion/converters/polarion_html_helper.py index 9d7fa573..fdd90ea1 100644 --- a/capella2polarion/converters/polarion_html_helper.py +++ b/capella2polarion/converters/polarion_html_helper.py @@ -124,7 +124,7 @@ def remove_table_ids( for element in html_fragments: if element.tag == "table": - element.remove("id") + element.attrib.pop("id", None) return html_fragments From 382c5932559a0de67c02792045c734705ffadd3d Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 30 Aug 2024 09:41:51 +0200 Subject: [PATCH 05/14] fix: use new client correctly --- capella2polarion/connectors/polarion_worker.py | 14 ++++++-------- tests/test_elements.py | 7 ++++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 03dfd012..c7d02356 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -127,21 +127,19 @@ def delete_orphaned_work_items( are marked as ``to be deleted`` via the status attribute. """ - def serialize_for_delete(uuid: str) -> str: - work_item_id, _ = self.polarion_data_repo[uuid] - logger.info("Delete work item %r...", work_item_id) - return work_item_id - existing_work_items = { uuid for uuid, _, work_item in self.polarion_data_repo.items() if work_item.status != "deleted" } uuids: set[str] = existing_work_items - set(converter_session) - work_item_ids = [serialize_for_delete(uuid) for uuid in uuids] - if work_item_ids: + work_items = [ + self.polarion_data_repo.get_work_item_by_capella_uuid(uuid) + for uuid in uuids + ] + if work_items: try: - self.project_client.work_items.delete(work_item_ids) + self.project_client.work_items.delete(work_items) self.polarion_data_repo.remove_work_items_by_capella_uuid( uuids ) diff --git a/tests/test_elements.py b/tests/test_elements.py index 7a1fffd4..95f94dd2 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -263,9 +263,10 @@ def test_delete_diagrams(diagr_base_object: BaseObjectContainer): pw.delete_orphaned_work_items(diagr_base_object.mc.converter_session) assert pw.project_client is not None assert pw.project_client.work_items.delete.call_count == 1 - assert pw.project_client.work_items.delete.call_args[0][0] == [ - "Diag-1" - ] + assert ( + pw.project_client.work_items.delete.call_args[0][0][0].id + == "Diag-1" + ) assert pw.project_client.work_items.create.call_count == 0 From 0125dfff7c18bccddeb59bce7d26cdf63c32185c Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Mon, 2 Sep 2024 23:06:38 +0200 Subject: [PATCH 06/14] feat: add enhancement to support the creation of text workitems - WIP --- capella2polarion/__main__.py | 1 - .../connectors/polarion_worker.py | 97 ++++++-- .../converters/document_config.py | 24 +- .../converters/document_renderer.py | 222 +++++++++++++----- .../converters/polarion_html_helper.py | 100 +++++++- capella2polarion/data_models.py | 14 ++ tests/test_cli.py | 12 - tests/test_documents.py | 161 ++++++++----- 8 files changed, 462 insertions(+), 169 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 728568b8..437588b3 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -171,7 +171,6 @@ def render_documents( for project, project_data in projects_document_data.items(): polarion_worker.post_documents(project_data.new_docs, project) polarion_worker.update_documents(project_data.updated_docs, project) - polarion_worker.update_headings(project_data.work_items, project) if __name__ == "__main__": diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index c7d02356..763f2db5 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -15,7 +15,11 @@ from capella2polarion import data_models from capella2polarion.connectors import polarion_repo -from capella2polarion.converters import data_session +from capella2polarion.converters import ( + data_session, + document_config, + polarion_html_helper, +) logger = logging.getLogger(__name__) @@ -25,6 +29,14 @@ int: 0, bool: False, } +WORK_ITEMS_IN_PROJECT_QUERY = ( + "SQL:(SELECT item.* FROM POLARION.WORKITEM item, POLARION.MODULE doc, " + "POLARION.PROJECT proj WHERE proj.C_ID = '{project}' AND " + "doc.FK_PROJECT = proj.C_PK AND doc.C_ID = '{doc_name}' AND " + "doc.C_MODULEFOLDER = '{doc_folder}' AND item.C_TYPE = '{wi_type}' AND " + "EXISTS (SELECT rel1.* FROM POLARION.REL_MODULE_WORKITEM rel1 WHERE " + "rel1.FK_URI_MODULE = doc.C_URI AND rel1.FK_URI_WORKITEM = item.C_URI))" +) class PolarionWorkerParams: @@ -455,22 +467,49 @@ def compare_and_update_work_items( def post_documents( self, - documents: list[polarion_api.Document], + document_datas: list[data_models.DocumentData], document_project: str | None = None, ): """Create new documents.""" client = self._get_client(document_project) + documents, _ = self._process_document_datas(client, document_datas) + client.documents.create(documents) def update_documents( self, - documents: list[polarion_api.Document], + document_datas: list[data_models.DocumentData], document_project: str | None = None, ): """Update existing documents.""" client = self._get_client(document_project) + documents, headings = self._process_document_datas( + client, document_datas + ) + + client.work_items.update(headings) client.documents.update(documents) + def _process_document_datas(self, client, document_datas): + documents = [] + headings = [] + for document_data in document_datas: + headings += document_data.headings + documents.append(document_data.document) + if document_data.text_work_items: + text_work_item_type = next( + iter(document_data.text_work_items.values()) + ).type + self._create_and_update_text_work_items( + document_data.text_work_items, client + ) + polarion_html_helper.insert_text_work_items( + document_data.document, + document_data.text_work_items, + text_work_item_type, + ) + return documents, headings + def get_document( self, space: str, name: str, document_project: str | None = None ) -> polarion_api.Document | None: @@ -485,23 +524,45 @@ def get_document( return None raise e - def update_headings( - self, - work_items: list[polarion_api.WorkItem], - document_project: str | None = None, - ): - """Update the given workitems without any additional checks.""" - client = self._get_client(document_project) - client.work_items.update(work_items) - def load_polarion_documents( self, - document_paths: t.Iterable[tuple[str | None, str, str]], - ) -> dict[tuple[str | None, str, str], polarion_api.Document | None]: - """Load the given document references from Polarion.""" + document_infos: t.Iterable[document_config.DocumentInfo], + ) -> dict[ + tuple[str | None, str, str], + tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], + ]: + """Load the documents referenced and text work items from Polarion.""" return { - (document_project, space, name): self.get_document( - space, name, document_project + (di.project_id, di.module_folder, di.module_name): ( + self.get_document( + di.module_folder, di.module_name, di.project_id + ), + self._get_client(di.project_id).work_items.get_all( + WORK_ITEMS_IN_PROJECT_QUERY.format( + project=di.project_id + or self.polarion_params.project_id, + doc_folder=di.module_folder, + doc_name=di.module_name, + wi_type=di.text_work_item_type, + ), + fields={"workitems": f"id,{di.text_work_item_id_field}"}, + ), ) - for document_project, space, name in document_paths + for di in document_infos } + + def _create_and_update_text_work_items( + self, + work_items: dict[str, polarion_api.WorkItem], + client: polarion_api.ProjectClient, + ): + client.work_items.update( + [work_item for work_item in work_items.values() if work_item.id] + ) + client.work_items.create( + [ + work_item + for work_item in work_items.values() + if not work_item.id + ] + ) diff --git a/capella2polarion/converters/document_config.py b/capella2polarion/converters/document_config.py index ad039924..be1df534 100644 --- a/capella2polarion/converters/document_config.py +++ b/capella2polarion/converters/document_config.py @@ -1,6 +1,7 @@ # Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 """Module with classes and a loader for document rendering configs.""" +import dataclasses import logging import pathlib import typing as t @@ -16,6 +17,17 @@ logger = logging.getLogger(__name__) +@dataclasses.dataclass +class DocumentInfo: + """Class for information regarding a document which should be created.""" + + project_id: str | None + module_folder: str + module_name: str + text_work_item_type: str + text_work_item_id_field: str + + class WorkItemLayout(pydantic.BaseModel): """Configuration for rendering layouts of work items.""" @@ -48,6 +60,8 @@ class BaseDocumentRenderingConfig(pydantic.BaseModel): template_directory: str | pathlib.Path project_id: str | None = None + text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE + text_work_item_id_field: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD status_allow_list: list[str] | None = None heading_numbering: bool = False work_item_layouts: dict[str, WorkItemLayout] = pydantic.Field( @@ -79,11 +93,17 @@ class DocumentConfigs(pydantic.BaseModel): pydantic.Field(default_factory=list) ) - def iterate_documents(self) -> t.Iterator[tuple[str | None, str, str]]: + def iterate_documents(self) -> t.Iterator[DocumentInfo]: """Yield all document paths of the config as tuples.""" for conf in self.full_authority + self.mixed_authority: for inst in conf.instances: - yield conf.project_id, inst.polarion_space, inst.polarion_name + yield DocumentInfo( + project_id=conf.project_id, + module_folder=inst.polarion_space, + module_name=inst.polarion_name, + text_work_item_type=conf.text_work_item_type, + text_work_item_id_field=conf.text_work_item_id_field, + ) def read_config_file( diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index 31ae4bfc..6b2ccb25 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -16,6 +16,7 @@ from capella2polarion.connectors import polarion_repo +from .. import data_models from . import document_config, polarion_html_helper logger = logging.getLogger(__name__) @@ -35,19 +36,19 @@ class RenderingSession: inserted_work_items: list[polarion_api.WorkItem] = dataclasses.field( default_factory=list ) + text_work_items: dict[str, polarion_api.WorkItem] = dataclasses.field( + default_factory=dict + ) @dataclasses.dataclass class ProjectData: """A class holding data of a project which documents are rendered for.""" - new_docs: list[polarion_api.Document] = dataclasses.field( - default_factory=list - ) - updated_docs: list[polarion_api.Document] = dataclasses.field( + new_docs: list[data_models.DocumentData] = dataclasses.field( default_factory=list ) - work_items: list[polarion_api.WorkItem] = dataclasses.field( + updated_docs: list[data_models.DocumentData] = dataclasses.field( default_factory=list ) @@ -68,6 +69,10 @@ def __init__( self.overwrite_heading_numbering = overwrite_heading_numbering self.overwrite_layouts = overwrite_layouts self.projects: dict[str | None, ProjectData] = {} + self.existing_documents: dict[ + tuple[str | None, str, str], + tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], + ] = {} def setup_env(self, env: jinja2.Environment): """Add globals and filters to the environment.""" @@ -95,22 +100,10 @@ def __insert_work_item( ) return f"

{self.__link_work_item(obj)}

" - layout_index = 0 - for layout in session.rendering_layouts: - if layout.type == wi.type: - break - layout_index += 1 - - if layout_index >= len(session.rendering_layouts): - session.rendering_layouts.append( - polarion_api.RenderingLayout( - type=wi.type, - layouter="section", - label=polarion_html_helper.camel_case_to_words( - wi.type - ), - ) - ) + assert wi.type + layout_index = polarion_html_helper.get_layout_index( + "section", session.rendering_layouts, wi.type + ) custom_info = "" if level is not None: @@ -147,7 +140,7 @@ def __heading(self, level: int, text: str, session: RenderingSession): session.headings.append(polarion_api.WorkItem(id=hid, title=text)) return ( f"' + f'id="{polarion_html_helper.wi_id_prefix}{hid}">' f"" ) return f"{text}" @@ -175,6 +168,9 @@ def render_document( document_title: str | None = None, heading_numbering: bool = False, rendering_layouts: list[polarion_api.RenderingLayout] | None = None, + *, + text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, + text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, **kwargs: t.Any, ): """Render a new Polarion document.""" @@ -186,6 +182,9 @@ def render_document( template_name: str, *, document: polarion_api.Document, + text_work_items: dict[str, polarion_api.WorkItem], + text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, + text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, **kwargs: t.Any, ): """Update an existing Polarion document.""" @@ -200,9 +199,13 @@ def render_document( heading_numbering: bool = False, rendering_layouts: list[polarion_api.RenderingLayout] | None = None, document: polarion_api.Document | None = None, + text_work_items: dict[str, polarion_api.WorkItem] | None = None, + text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, + text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, **kwargs: t.Any, ): """Render a Polarion document.""" + text_work_items = text_work_items or {} if document is not None: polarion_folder = document.module_folder polarion_name = document.module_name @@ -232,13 +235,25 @@ def render_document( if rendering_layouts is not None: session.rendering_layouts = rendering_layouts + rendering_result = template.render( + model=self.model, session=session, **kwargs + ) + new_text_work_items = self._extract_text_work_items( + lxmlhtml.fragments_fromstring(rendering_result), + text_work_items, + text_work_item_type, + text_work_item_identifier, + ) + document.home_page_content = polarion_api.TextContent( "text/html", - template.render(model=self.model, session=session, **kwargs), + rendering_result, ) document.rendering_layouts = session.rendering_layouts - return document, session.headings + return data_models.DocumentData( + document, session.headings, new_text_work_items + ) def update_mixed_authority_document( self, @@ -247,6 +262,9 @@ def update_mixed_authority_document( sections: dict[str, str], global_parameters: dict[str, t.Any], section_parameters: dict[str, dict[str, t.Any]], + text_work_items: dict[str, polarion_api.WorkItem], + text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, + text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, ): """Update a mixed authority document.""" assert ( @@ -263,6 +281,7 @@ def update_mixed_authority_document( env = self._get_jinja_env(template_folder) new_content = [] + new_text_work_items = {} last_section_end = 0 for section_name, area in section_areas.items(): @@ -288,7 +307,24 @@ def update_mixed_authority_document( | section_parameters.get(section_name, {}) ), ) - new_content += lxmlhtml.fragments_fromstring(content) + work_item_ids = polarion_html_helper.extract_work_items( + current_content + ) + section_text_work_items = { + text_id: work_item + for text_id, work_item in text_work_items.items() + if work_item.id in work_item_ids + } + html_fragments = lxmlhtml.fragments_fromstring(content) + new_text_work_items.update( + self._extract_text_work_items( + html_fragments, + section_text_work_items, + text_work_item_type, + text_work_item_identifier, + ) + ) + new_content += html_fragments new_content += html_elements[last_section_end:] new_content = polarion_html_helper.remove_table_ids(new_content) @@ -304,7 +340,9 @@ def update_mixed_authority_document( ) document.rendering_layouts = session.rendering_layouts - return document, session.headings + return data_models.DocumentData( + document, session.headings, new_text_work_items + ) def _get_and_customize_doc( self, @@ -314,11 +352,11 @@ def _get_and_customize_doc( title: str | None, rendering_layouts: list[polarion_api.RenderingLayout], heading_numbering: bool, - existing_documents: dict[ - tuple[str | None, str, str], polarion_api.Document | None - ], - ) -> polarion_api.Document | None: - if old_doc := existing_documents.get((project_id, space, name)): + ) -> tuple[polarion_api.Document | None, list[polarion_api.WorkItem]]: + old_doc, text_work_items = self.existing_documents.get( + (project_id, space, name), (None, []) + ) + if old_doc: if title: old_doc.title = title if self.overwrite_layouts: @@ -326,13 +364,14 @@ def _get_and_customize_doc( if self.overwrite_heading_numbering: old_doc.outline_numbering = heading_numbering - return old_doc + return old_doc, text_work_items def render_documents( self, configs: document_config.DocumentConfigs, existing_documents: dict[ - tuple[str | None, str, str], polarion_api.Document | None + tuple[str | None, str, str], + tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], ], ) -> dict[str | None, ProjectData]: """Render all documents defined in the given config. @@ -340,18 +379,33 @@ def render_documents( Returns a list new documents followed by updated documents and work items, which need to be updated """ + self.existing_documents = existing_documents + self.projects = {} - self._render_full_authority_documents( - configs.full_authority, - existing_documents, - ) - - self._render_mixed_authority_documents( - configs.mixed_authority, existing_documents - ) + self._render_full_authority_documents(configs.full_authority) + self._render_mixed_authority_documents(configs.mixed_authority) return self.projects + def _make_text_work_item_mapping( + self, + work_items: list[polarion_api.WorkItem], + text_work_item_field_id: str, + ) -> dict[str, polarion_api.WorkItem]: + result = {} + for work_item in work_items: + # We only use those work items which have an ID defined by us + if text_id := work_item.additional_attributes.get( + text_work_item_field_id + ): + if text_id in result: + raise ValueError( + f"There are multiple text work items with {text_work_item_field_id} == {text_id}" + ) + + result[text_id] = work_item + return result + def _check_document_status( self, document: polarion_api.Document, @@ -375,10 +429,7 @@ def _check_document_status( def _render_mixed_authority_documents( self, mixed_authority_configs: list[ - document_config.FullAuthorityDocumentRenderingConfig - ], - existing_documents: dict[ - tuple[str | None, str, str], polarion_api.Document | None + document_config.MixedAuthorityDocumentRenderingConfig ], ): for config in mixed_authority_configs: @@ -389,14 +440,13 @@ def _render_mixed_authority_documents( config.project_id, ProjectData() ) for instance in config.instances: - old_doc = self._get_and_customize_doc( + old_doc, text_work_items = self._get_and_customize_doc( config.project_id, instance.polarion_space, instance.polarion_name, instance.polarion_title, rendering_layouts, config.heading_numbering, - existing_documents, ) if old_doc is None: logger.error( @@ -411,12 +461,17 @@ def _render_mixed_authority_documents( continue try: - new_doc, wis = self.update_mixed_authority_document( + document_data = self.update_mixed_authority_document( old_doc, config.template_directory, config.sections, instance.params, instance.section_params, + self._make_text_work_item_mapping( + text_work_items, config.text_work_item_id_field + ), + config.text_work_item_id_field, + config.text_work_item_type, ) except Exception as e: logger.error( @@ -428,15 +483,11 @@ def _render_mixed_authority_documents( ) continue - project_data.updated_docs.append(new_doc) - project_data.work_items.extend(wis) + project_data.updated_docs.append(document_data) def _render_full_authority_documents( self, full_authority_configs, - existing_documents: dict[ - tuple[str | None, str, str], polarion_api.Document | None - ], ): for config in full_authority_configs: rendering_layouts = document_config.generate_work_item_layouts( @@ -446,23 +497,28 @@ def _render_full_authority_documents( config.project_id, ProjectData() ) for instance in config.instances: - if old_doc := self._get_and_customize_doc( + old_doc, text_work_items = self._get_and_customize_doc( config.project_id, instance.polarion_space, instance.polarion_name, instance.polarion_title, rendering_layouts, config.heading_numbering, - existing_documents, - ): + ) + if old_doc: if not self._check_document_status(old_doc, config): continue try: - new_doc, wis = self.render_document( + document_data = self.render_document( config.template_directory, config.template, document=old_doc, + text_work_items=self._make_text_work_item_mapping( + text_work_items, config.text_work_item_id_field + ), + text_work_item_identifier=config.text_work_item_id_field, + text_work_item_type=config.text_work_item_type, **instance.params, ) except Exception as e: @@ -475,11 +531,10 @@ def _render_full_authority_documents( ) continue - project_data.updated_docs.append(new_doc) - project_data.work_items.extend(wis) + project_data.updated_docs.append(document_data) else: try: - new_doc, _ = self.render_document( + document_data = self.render_document( config.template_directory, config.template, instance.polarion_space, @@ -487,6 +542,8 @@ def _render_full_authority_documents( instance.polarion_title, config.heading_numbering, rendering_layouts, + text_work_item_identifier=config.text_work_item_id_field, + text_work_item_type=config.text_work_item_type, **instance.params, ) except Exception as e: @@ -499,7 +556,7 @@ def _render_full_authority_documents( ) continue - project_data.new_docs.append(new_doc) + project_data.new_docs.append(document_data) def _extract_section_areas(self, html_elements: list[etree._Element]): section_areas = {} @@ -546,3 +603,46 @@ def _extract_section_areas(self, html_elements: list[etree._Element]): current_area_id = None current_area_start = None return section_areas + + def _extract_text_work_items( + self, + content: list[lxmlhtml.HtmlElement], + text_work_items: dict[str, polarion_api.WorkItem], + text_work_item_type: str, + field_id: str, + ) -> dict[str, polarion_api.WorkItem]: + work_items: dict[str, polarion_api.WorkItem] = {} + for element in content: + if element.tag != polarion_html_helper.WORK_ITEM_TAG: + continue + + if not (text_id := element.get("id")): + raise ValueError("All work items must have an ID in template") + + work_item = text_work_items.pop( + text_id, + polarion_api.WorkItem( + type=text_work_item_type, + title="", + status="open", + additional_attributes={field_id: text_id}, + ), + ) + work_item.description_type = "text/html" + inner_content = "".join( + [ + ( + lxmlhtml.tostring(child, encoding="unicode") + if isinstance(child, lxmlhtml.HtmlElement) + else child + ) + for child in element.iterchildren() + ] + ) + if element.text: + inner_content = element.text + inner_content + + work_item.description = inner_content + work_items[text_id] = work_item + + return work_items diff --git a/capella2polarion/converters/polarion_html_helper.py b/capella2polarion/converters/polarion_html_helper.py index fdd90ea1..f24a7913 100644 --- a/capella2polarion/converters/polarion_html_helper.py +++ b/capella2polarion/converters/polarion_html_helper.py @@ -8,14 +8,16 @@ import capellambse import jinja2 +import polarion_rest_api_client as polarion_api from capellambse import helpers as chelpers from lxml import etree, html -heading_id_prefix = "polarion_wiki macro name=module-workitem;params=id=" +wi_id_prefix = "polarion_wiki macro name=module-workitem;params=id=" h_regex = re.compile("h[0-9]") -wi_regex = re.compile(f"{heading_id_prefix}(.*)") - +wi_id_regex = re.compile(f"{wi_id_prefix}([A-Z|a-z|0-9]*-[0-9]+)") +TEXT_WORK_ITEM_ID_FIELD = "__C2P__id" +TEXT_WORK_ITEM_TYPE = "text" POLARION_WORK_ITEM_URL = ( '' @@ -29,6 +31,7 @@ f"<deleted element ({chelpers.RE_VALID_UUID.pattern})>" ) RED_TEXT = '

{text}

' +WORK_ITEM_TAG = "workitem" def strike_through(string: str) -> str: @@ -111,7 +114,7 @@ def setup_env(self, env: jinja2.Environment): def remove_table_ids( - html_content: str | list[etree._Element], + html_content: str | list[html.HtmlComment], ) -> list[etree._Element]: """Remove the ID field from all tables. @@ -130,24 +133,95 @@ def remove_table_ids( def _ensure_fragments( - html_content: str | list[etree._Element], -) -> list[etree._Element]: + html_content: str | list[html.HtmlComment], +) -> list[html.HtmlComment]: if isinstance(html_content, str): return html.fragments_fromstring(html_content) return html_content -def extract_headings(html_content: str | list[etree._Element]) -> list[str]: +def extract_headings(html_content: str | list[html.HtmlComment]) -> list[str]: """Return a list of work item IDs for all headings in the given content.""" - heading_ids = [] - html_fragments = _ensure_fragments(html_content) + return extract_work_items(html_content, h_regex) + +def extract_work_items( + html_content: str | list[html.HtmlComment], + tag_regex: re.Pattern | None = None, +) -> list[str]: + """Return a list of work item IDs for work items in the given content.""" + work_items = [] + html_fragments = _ensure_fragments(html_content) for element in html_fragments: if isinstance(element, html.HtmlComment): continue - if h_regex.fullmatch(element.tag): - if matches := wi_regex.match(element.get("id")): - heading_ids.append(matches.group(1)) + if (tag_regex is not None and tag_regex.fullmatch(element.tag)) or ( + tag_regex is None and element.tag == "div" + ): + if matches := wi_id_regex.match(element.get("id")): + work_items.append(matches.group(1)) + return work_items + + +def insert_text_work_items( + document: polarion_api.Document, + text_work_items: dict[str, polarion_api.WorkItem], + text_work_item_type: str, +): + """Insert text work items into the given document.""" + if not text_work_items: + return + + assert document.home_page_content is not None + layout_index = get_layout_index( + "paragraph", document.rendering_layouts, text_work_item_type + ) + html_fragments = _ensure_fragments(document.home_page_content.value) + new_content = [] + last_match = -1 + for index, element in enumerate(html_fragments): + if isinstance(element, html.HtmlComment): + continue + + if element.tag == "workitem": + new_content += html_fragments[last_match + 1 : index] + last_match = index + if work_item := text_work_items.get(element.get("id")): + new_content.append( + html.fromstring( + POLARION_WORK_ITEM_DOCUMENT.format( + pid=work_item.id, lid=layout_index, custom_info="" + ) + ) + ) + + new_content += html_fragments[last_match + 1 :] + document.home_page_content.value = "\n".join( + [html.tostring(element).decode("utf-8") for element in new_content] + ) + - return heading_ids +def get_layout_index( + default_layouter: str, + rendering_layouts: list[polarion_api.RenderingLayout], + work_item_type: str, +) -> int: + """Return the index of the layout of the requested workitem. + + If there is no rendering config yet, it will be created. + """ + layout_index = 0 + for layout in rendering_layouts: + if layout.type == work_item_type: + return layout_index + layout_index += 1 + if layout_index >= len(rendering_layouts): + rendering_layouts.append( + polarion_api.RenderingLayout( + type=work_item_type, + layouter=default_layouter, + label=camel_case_to_words(work_item_type), + ) + ) + return layout_index diff --git a/capella2polarion/data_models.py b/capella2polarion/data_models.py index 3b445795..9ef8153a 100644 --- a/capella2polarion/data_models.py +++ b/capella2polarion/data_models.py @@ -4,6 +4,7 @@ from __future__ import annotations import base64 +import dataclasses import hashlib import json import typing as t @@ -60,3 +61,16 @@ def calculate_checksum(self) -> str: | dict(sorted(attachment_checksums.items())) ) return self._checksum + + +@dataclasses.dataclass +class DocumentData: + """A class to store data related to a rendered document.""" + + document: polarion_api.Document + headings: list[polarion_api.WorkItem] = dataclasses.field( + default_factory=list + ) + text_work_items: dict[str, polarion_api.WorkItem] = dataclasses.field( + default_factory=dict + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index b384e75e..b102592a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -129,12 +129,6 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): "update_documents", mock_update_documents, ) - mock_update_headings = mock.MagicMock() - monkeypatch.setattr( - polarion_worker.CapellaPolarionWorker, - "update_headings", - mock_update_headings, - ) command: list[str] = [ "--polarion-project-id", @@ -178,9 +172,3 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): assert len(mock_update_documents.call_args_list[1].args[0]) == 0 assert mock_update_documents.call_args_list[0].args[1] is None assert mock_update_documents.call_args_list[1].args[1] == "TestProject" - - assert mock_update_headings.call_count == 2 - assert len(mock_update_headings.call_args_list[0].args[0]) == 1 - assert len(mock_update_headings.call_args_list[1].args[0]) == 0 - assert mock_update_headings.call_args_list[0].args[1] is None - assert mock_update_headings.call_args_list[1].args[1] == "TestProject" diff --git a/tests/test_documents.py b/tests/test_documents.py index 4eacbd89..47f33569 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -19,50 +19,63 @@ MIXED_AUTHORITY_DOCUMENT = TEST_DOCUMENT_ROOT / "mixed_authority_doc.html" -def existing_documents() -> ( - dict[tuple[str | None, str, str], polarion_api.Document] -): +def existing_documents() -> dict[ + tuple[str | None, str, str], + tuple[polarion_api.Document, list[polarion_api.WorkItem]], +]: return { - (None, "_default", "id123"): polarion_api.Document( - module_folder="_default", - module_name="id123", - status="draft", - home_page_content=polarion_api.TextContent( - type="text/html", - value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + (None, "_default", "id123"): ( + polarion_api.Document( + module_folder="_default", + module_name="id123", + status="draft", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), + rendering_layouts=[ + polarion_api.RenderingLayout( + "Class", "paragraph", type="class" + ) + ], ), - rendering_layouts=[ - polarion_api.RenderingLayout( - "Class", "paragraph", type="class" - ) - ], + [], ), - (None, "_default", "id1237"): polarion_api.Document( - module_folder="_default", - module_name="id1237", - status="draft", - home_page_content=polarion_api.TextContent( - type="text/html", - value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + (None, "_default", "id1237"): ( + polarion_api.Document( + module_folder="_default", + module_name="id1237", + status="draft", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), ), + [], ), - ("TestProject", "_default", "id1239"): polarion_api.Document( - module_folder="_default", - module_name="id1239", - status="in_review", - home_page_content=polarion_api.TextContent( - type="text/html", - value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ("TestProject", "_default", "id1239"): ( + polarion_api.Document( + module_folder="_default", + module_name="id1239", + status="in_review", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), ), + [], ), - ("TestProject", "_default", "id1240"): polarion_api.Document( - module_folder="_default", - module_name="id1240", - status="draft", - home_page_content=polarion_api.TextContent( - type="text/html", - value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ("TestProject", "_default", "id1240"): ( + polarion_api.Document( + module_folder="_default", + module_name="id1240", + status="draft", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), ), + [], ), } @@ -89,7 +102,7 @@ def test_create_new_document( empty_polarion_worker.polarion_data_repo, model ) - new_doc, wis = renderer.render_document( + document_data = renderer.render_document( JUPYTER_TEMPLATE_FOLDER, CLASSES_TEMPLATE, "_default", @@ -98,10 +111,10 @@ def test_create_new_document( ) content: list[etree._Element] = html.fromstring( - new_doc.home_page_content.value + document_data.document.home_page_content.value ) - assert len(wis) == 0 - assert new_doc.rendering_layouts == [ + assert len(document_data.headings) == 0 + assert document_data.document.rendering_layouts == [ polarion_api.RenderingLayout( label="Class", type="class", layouter="section" ) @@ -160,18 +173,19 @@ def test_update_document( ), ) - new_doc, wis = renderer.render_document( + document_data = renderer.render_document( JUPYTER_TEMPLATE_FOLDER, CLASSES_TEMPLATE, document=old_doc, + text_work_items={}, cls="c710f1c2-ede6-444e-9e2b-0ff30d7fd040", ) content: list[etree._Element] = html.fromstring( - new_doc.home_page_content.value + document_data.document.home_page_content.value ) - assert len(new_doc.rendering_layouts) == 1 - assert new_doc.rendering_layouts[ + assert len(document_data.document.rendering_layouts) == 1 + assert document_data.document.rendering_layouts[ 0 ].properties == polarion_api.data_models.RenderingProperties( fields_at_start=["ID"] @@ -182,9 +196,9 @@ def test_update_document( assert content[0].tag == "h1" assert content[1].text == "Data Classes" assert content[1].tag == "h2" - assert len(wis) == 1 - assert wis[0].id == "ATSY-16062" - assert wis[0].title == "Class Document" + assert len(document_data.headings) == 1 + assert document_data.headings[0].id == "ATSY-16062" + assert document_data.headings[0].title == "Class Document" def test_mixed_authority_document( @@ -202,7 +216,7 @@ def test_mixed_authority_document( ), ) - new_doc, wis = renderer.update_mixed_authority_document( + document_data = renderer.update_mixed_authority_document( old_doc, DOCUMENT_SECTIONS, { @@ -217,10 +231,11 @@ def test_mixed_authority_document( "global_param": "Overwrite global param", }, }, + {}, ) content: list[etree._Element] = html.fromstring( - new_doc.home_page_content.value + document_data.document.home_page_content.value ) assert len(content) == 15 @@ -236,9 +251,9 @@ def test_mixed_authority_document( assert content[11].text == "Overwritten: Overwrite global param" assert content[12].text == "Local Test section 2" assert content[14].text == "Some postfix stuff" - assert len(wis) == 1 - assert wis[0].id == "ATSY-18305" - assert wis[0].title == "Keep Heading" + assert len(document_data.headings) == 1 + assert document_data.headings[0].id == "ATSY-18305" + assert document_data.headings[0].title == "Keep Heading" def test_render_all_documents_partially_successfully( @@ -268,12 +283,34 @@ def test_render_all_documents_partially_successfully( # In all existing documents we had 2 headings. In full authority mode # both should be updated and in mixed authority mode only one of them as # the other is outside the rendering area - assert len(projects_data[None].work_items) == 3 - assert len(projects_data["TestProject"].work_items) == 2 - assert len(projects_data[None].updated_docs[0].rendering_layouts) == 0 - assert len(projects_data[None].updated_docs[1].rendering_layouts) == 1 - assert projects_data[None].updated_docs[0].outline_numbering is None - assert projects_data[None].updated_docs[1].outline_numbering is None + assert ( + sum( + len(document_data.headings) + for document_data in projects_data[None].updated_docs + ) + == 3 + ) + assert ( + sum( + len(document_data.headings) + for document_data in projects_data["TestProject"].updated_docs + ) + == 2 + ) + assert ( + len(projects_data[None].updated_docs[0].document.rendering_layouts) + == 0 + ) + assert ( + len(projects_data[None].updated_docs[1].document.rendering_layouts) + == 1 + ) + assert ( + projects_data[None].updated_docs[0].document.outline_numbering is None + ) + assert ( + projects_data[None].updated_docs[1].document.outline_numbering is None + ) def test_render_all_documents_overwrite_headings_layouts( @@ -290,10 +327,10 @@ def test_render_all_documents_overwrite_headings_layouts( projects_data = renderer.render_documents(conf, existing_documents()) updated_docs = projects_data[None].updated_docs - assert len(updated_docs[0].rendering_layouts) == 2 - assert len(updated_docs[1].rendering_layouts) == 2 - assert updated_docs[0].outline_numbering is False - assert updated_docs[1].outline_numbering is False + assert len(updated_docs[0].document.rendering_layouts) == 2 + assert len(updated_docs[1].document.rendering_layouts) == 2 + assert updated_docs[0].document.outline_numbering is False + assert updated_docs[1].document.outline_numbering is False def test_full_authority_document_config(): From 398a536e89aa5228bc5f966793d89b44b69c31c8 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 3 Sep 2024 09:41:32 +0200 Subject: [PATCH 07/14] refactor: refactor text work item generation --- .../connectors/polarion_worker.py | 20 +-- .../converters/document_config.py | 17 +-- .../converters/document_renderer.py | 142 ++++-------------- .../converters/polarion_html_helper.py | 17 ++- .../converters/text_work_item_provider.py | 122 +++++++++++++++ capella2polarion/data_models.py | 21 ++- tests/test_documents.py | 1 - 7 files changed, 188 insertions(+), 152 deletions(-) create mode 100644 capella2polarion/converters/text_work_item_provider.py diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 763f2db5..a1a348eb 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -15,11 +15,7 @@ from capella2polarion import data_models from capella2polarion.connectors import polarion_repo -from capella2polarion.converters import ( - data_session, - document_config, - polarion_html_helper, -) +from capella2polarion.converters import data_session, polarion_html_helper logger = logging.getLogger(__name__) @@ -496,17 +492,13 @@ def _process_document_datas(self, client, document_datas): for document_data in document_datas: headings += document_data.headings documents.append(document_data.document) - if document_data.text_work_items: - text_work_item_type = next( - iter(document_data.text_work_items.values()) - ).type + if document_data.text_work_item_provider.new_text_work_items: self._create_and_update_text_work_items( - document_data.text_work_items, client + document_data.text_work_item_provider.new_text_work_items, + client, ) - polarion_html_helper.insert_text_work_items( + document_data.text_work_item_provider.insert_text_work_items( document_data.document, - document_data.text_work_items, - text_work_item_type, ) return documents, headings @@ -526,7 +518,7 @@ def get_document( def load_polarion_documents( self, - document_infos: t.Iterable[document_config.DocumentInfo], + document_infos: t.Iterable[data_models.DocumentInfo], ) -> dict[ tuple[str | None, str, str], tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], diff --git a/capella2polarion/converters/document_config.py b/capella2polarion/converters/document_config.py index be1df534..6a936666 100644 --- a/capella2polarion/converters/document_config.py +++ b/capella2polarion/converters/document_config.py @@ -1,7 +1,6 @@ # Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 """Module with classes and a loader for document rendering configs.""" -import dataclasses import logging import pathlib import typing as t @@ -12,22 +11,12 @@ import pydantic import yaml +from capella2polarion import data_models from capella2polarion.converters import polarion_html_helper logger = logging.getLogger(__name__) -@dataclasses.dataclass -class DocumentInfo: - """Class for information regarding a document which should be created.""" - - project_id: str | None - module_folder: str - module_name: str - text_work_item_type: str - text_work_item_id_field: str - - class WorkItemLayout(pydantic.BaseModel): """Configuration for rendering layouts of work items.""" @@ -93,11 +82,11 @@ class DocumentConfigs(pydantic.BaseModel): pydantic.Field(default_factory=list) ) - def iterate_documents(self) -> t.Iterator[DocumentInfo]: + def iterate_documents(self) -> t.Iterator[data_models.DocumentInfo]: """Yield all document paths of the config as tuples.""" for conf in self.full_authority + self.mixed_authority: for inst in conf.instances: - yield DocumentInfo( + yield data_models.DocumentInfo( project_id=conf.project_id, module_folder=inst.polarion_space, module_name=inst.polarion_name, diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index 6b2ccb25..a0c1766a 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -18,6 +18,7 @@ from .. import data_models from . import document_config, polarion_html_helper +from . import text_work_item_provider as twi logger = logging.getLogger(__name__) @@ -169,10 +170,9 @@ def render_document( heading_numbering: bool = False, rendering_layouts: list[polarion_api.RenderingLayout] | None = None, *, - text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, - text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, + text_work_item_provider: twi.TextWorkItemProvider | None = None, **kwargs: t.Any, - ): + ) -> data_models.DocumentData: """Render a new Polarion document.""" @t.overload @@ -182,11 +182,9 @@ def render_document( template_name: str, *, document: polarion_api.Document, - text_work_items: dict[str, polarion_api.WorkItem], - text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, - text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, + text_work_item_provider: twi.TextWorkItemProvider | None = None, **kwargs: t.Any, - ): + ) -> data_models.DocumentData: """Update an existing Polarion document.""" def render_document( @@ -199,13 +197,13 @@ def render_document( heading_numbering: bool = False, rendering_layouts: list[polarion_api.RenderingLayout] | None = None, document: polarion_api.Document | None = None, - text_work_items: dict[str, polarion_api.WorkItem] | None = None, - text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, - text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, + text_work_item_provider: twi.TextWorkItemProvider | None = None, **kwargs: t.Any, - ): + ) -> data_models.DocumentData: """Render a Polarion document.""" - text_work_items = text_work_items or {} + text_work_item_provider = ( + text_work_item_provider or twi.TextWorkItemProvider() + ) if document is not None: polarion_folder = document.module_folder polarion_name = document.module_name @@ -238,11 +236,8 @@ def render_document( rendering_result = template.render( model=self.model, session=session, **kwargs ) - new_text_work_items = self._extract_text_work_items( + text_work_item_provider.generate_text_work_items( lxmlhtml.fragments_fromstring(rendering_result), - text_work_items, - text_work_item_type, - text_work_item_identifier, ) document.home_page_content = polarion_api.TextContent( @@ -252,7 +247,7 @@ def render_document( document.rendering_layouts = session.rendering_layouts return data_models.DocumentData( - document, session.headings, new_text_work_items + document, session.headings, text_work_item_provider ) def update_mixed_authority_document( @@ -262,11 +257,12 @@ def update_mixed_authority_document( sections: dict[str, str], global_parameters: dict[str, t.Any], section_parameters: dict[str, dict[str, t.Any]], - text_work_items: dict[str, polarion_api.WorkItem], - text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, - text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, - ): + text_work_item_provider: twi.TextWorkItemProvider | None = None, + ) -> data_models.DocumentData: """Update a mixed authority document.""" + text_work_item_provider = ( + text_work_item_provider or twi.TextWorkItemProvider() + ) assert ( document.home_page_content and document.home_page_content.value ), "In mixed authority the document must have content" @@ -281,7 +277,6 @@ def update_mixed_authority_document( env = self._get_jinja_env(template_folder) new_content = [] - new_text_work_items = {} last_section_end = 0 for section_name, area in section_areas.items(): @@ -310,19 +305,9 @@ def update_mixed_authority_document( work_item_ids = polarion_html_helper.extract_work_items( current_content ) - section_text_work_items = { - text_id: work_item - for text_id, work_item in text_work_items.items() - if work_item.id in work_item_ids - } html_fragments = lxmlhtml.fragments_fromstring(content) - new_text_work_items.update( - self._extract_text_work_items( - html_fragments, - section_text_work_items, - text_work_item_type, - text_work_item_identifier, - ) + text_work_item_provider.generate_text_work_items( + html_fragments, work_item_ids ) new_content += html_fragments @@ -341,7 +326,7 @@ def update_mixed_authority_document( document.rendering_layouts = session.rendering_layouts return data_models.DocumentData( - document, session.headings, new_text_work_items + document, session.headings, text_work_item_provider ) def _get_and_customize_doc( @@ -387,25 +372,6 @@ def render_documents( return self.projects - def _make_text_work_item_mapping( - self, - work_items: list[polarion_api.WorkItem], - text_work_item_field_id: str, - ) -> dict[str, polarion_api.WorkItem]: - result = {} - for work_item in work_items: - # We only use those work items which have an ID defined by us - if text_id := work_item.additional_attributes.get( - text_work_item_field_id - ): - if text_id in result: - raise ValueError( - f"There are multiple text work items with {text_work_item_field_id} == {text_id}" - ) - - result[text_id] = work_item - return result - def _check_document_status( self, document: polarion_api.Document, @@ -448,6 +414,11 @@ def _render_mixed_authority_documents( rendering_layouts, config.heading_numbering, ) + text_work_item_provider = twi.TextWorkItemProvider( + config.text_work_item_id_field, + config.text_work_item_type, + text_work_items, + ) if old_doc is None: logger.error( "For document %s/%s no document was found, but it's " @@ -467,11 +438,7 @@ def _render_mixed_authority_documents( config.sections, instance.params, instance.section_params, - self._make_text_work_item_mapping( - text_work_items, config.text_work_item_id_field - ), - config.text_work_item_id_field, - config.text_work_item_type, + text_work_item_provider, ) except Exception as e: logger.error( @@ -505,6 +472,11 @@ def _render_full_authority_documents( rendering_layouts, config.heading_numbering, ) + text_work_item_provider = twi.TextWorkItemProvider( + config.text_work_item_id_field, + config.text_work_item_type, + text_work_items, + ) if old_doc: if not self._check_document_status(old_doc, config): continue @@ -514,11 +486,7 @@ def _render_full_authority_documents( config.template_directory, config.template, document=old_doc, - text_work_items=self._make_text_work_item_mapping( - text_work_items, config.text_work_item_id_field - ), - text_work_item_identifier=config.text_work_item_id_field, - text_work_item_type=config.text_work_item_type, + text_work_item_provider=text_work_item_provider, **instance.params, ) except Exception as e: @@ -542,8 +510,7 @@ def _render_full_authority_documents( instance.polarion_title, config.heading_numbering, rendering_layouts, - text_work_item_identifier=config.text_work_item_id_field, - text_work_item_type=config.text_work_item_type, + text_work_item_provider=text_work_item_provider, **instance.params, ) except Exception as e: @@ -603,46 +570,3 @@ def _extract_section_areas(self, html_elements: list[etree._Element]): current_area_id = None current_area_start = None return section_areas - - def _extract_text_work_items( - self, - content: list[lxmlhtml.HtmlElement], - text_work_items: dict[str, polarion_api.WorkItem], - text_work_item_type: str, - field_id: str, - ) -> dict[str, polarion_api.WorkItem]: - work_items: dict[str, polarion_api.WorkItem] = {} - for element in content: - if element.tag != polarion_html_helper.WORK_ITEM_TAG: - continue - - if not (text_id := element.get("id")): - raise ValueError("All work items must have an ID in template") - - work_item = text_work_items.pop( - text_id, - polarion_api.WorkItem( - type=text_work_item_type, - title="", - status="open", - additional_attributes={field_id: text_id}, - ), - ) - work_item.description_type = "text/html" - inner_content = "".join( - [ - ( - lxmlhtml.tostring(child, encoding="unicode") - if isinstance(child, lxmlhtml.HtmlElement) - else child - ) - for child in element.iterchildren() - ] - ) - if element.text: - inner_content = element.text + inner_content - - work_item.description = inner_content - work_items[text_id] = work_item - - return work_items diff --git a/capella2polarion/converters/polarion_html_helper.py b/capella2polarion/converters/polarion_html_helper.py index f24a7913..9c69cb18 100644 --- a/capella2polarion/converters/polarion_html_helper.py +++ b/capella2polarion/converters/polarion_html_helper.py @@ -123,7 +123,7 @@ def remove_table_ids( time the REST-API does not allow posting or patching a document with multiple tables having the same ID. """ - html_fragments = _ensure_fragments(html_content) + html_fragments = ensure_fragments(html_content) for element in html_fragments: if element.tag == "table": @@ -132,26 +132,27 @@ def remove_table_ids( return html_fragments -def _ensure_fragments( - html_content: str | list[html.HtmlComment], -) -> list[html.HtmlComment]: +def ensure_fragments( + html_content: str | list[html.HtmlElement], +) -> list[html.HtmlElement]: + """Convert string to html elements.""" if isinstance(html_content, str): return html.fragments_fromstring(html_content) return html_content -def extract_headings(html_content: str | list[html.HtmlComment]) -> list[str]: +def extract_headings(html_content: str | list[html.HtmlElement]) -> list[str]: """Return a list of work item IDs for all headings in the given content.""" return extract_work_items(html_content, h_regex) def extract_work_items( - html_content: str | list[html.HtmlComment], + html_content: str | list[html.HtmlElement], tag_regex: re.Pattern | None = None, ) -> list[str]: """Return a list of work item IDs for work items in the given content.""" work_items = [] - html_fragments = _ensure_fragments(html_content) + html_fragments = ensure_fragments(html_content) for element in html_fragments: if isinstance(element, html.HtmlComment): continue @@ -177,7 +178,7 @@ def insert_text_work_items( layout_index = get_layout_index( "paragraph", document.rendering_layouts, text_work_item_type ) - html_fragments = _ensure_fragments(document.home_page_content.value) + html_fragments = ensure_fragments(document.home_page_content.value) new_content = [] last_match = -1 for index, element in enumerate(html_fragments): diff --git a/capella2polarion/converters/text_work_item_provider.py b/capella2polarion/converters/text_work_item_provider.py new file mode 100644 index 00000000..cb74ae4f --- /dev/null +++ b/capella2polarion/converters/text_work_item_provider.py @@ -0,0 +1,122 @@ +# Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 +"""Provides a class to generate and inset text work items in documents.""" +import polarion_rest_api_client as polarion_api +from lxml import html + +from capella2polarion.converters import polarion_html_helper + + +class TextWorkItemProvider: + """Class providing text work items, their generation and insertion.""" + + def __init__( + self, + text_work_item_id_field: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, + text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, + existing_text_work_items: list[polarion_api.WorkItem] | None = None, + ): + self.old_text_work_items: dict[str, polarion_api.WorkItem] = {} + for work_item in existing_text_work_items or []: + # We only use those work items which have an ID defined by us + if text_id := work_item.additional_attributes.get( + text_work_item_id_field + ): + if text_id in self.old_text_work_items: + raise ValueError( + f"There are multiple text work items with " + f"{text_work_item_id_field} == {text_id}" + ) + + self.old_text_work_items[text_id] = work_item + + self.text_work_item_id_field = text_work_item_id_field + self.text_work_item_type = text_work_item_type + self.new_text_work_items: dict[str, polarion_api.WorkItem] = {} + + def generate_text_work_items( + self, + content: list[html.HtmlElement] | str, + work_item_id_filter: list[str] | None = None, + ): + """Generate text work items from the provided html.""" + content = polarion_html_helper.ensure_fragments(content) + for element in content: + if element.tag != polarion_html_helper.WORK_ITEM_TAG: + continue + + if not (text_id := element.get("id")): + raise ValueError("All work items must have an ID in template") + + if ( + work_item_id_filter is None or text_id in work_item_id_filter + ) and text_id in self.old_text_work_items: + work_item = self.old_text_work_items[text_id] + else: + work_item = polarion_api.WorkItem( + type=self.text_work_item_type, + title="", + status="open", + additional_attributes={ + self.text_work_item_id_field: text_id + }, + ) + + work_item.description_type = "text/html" + inner_content = "".join( + [ + ( + html.tostring(child, encoding="unicode") + if isinstance(child, html.HtmlElement) + else child + ) + for child in element.iterchildren() + ] + ) + if element.text: + inner_content = element.text + inner_content + + work_item.description = inner_content + self.new_text_work_items[text_id] = work_item + + def insert_text_work_items( + self, + document: polarion_api.Document, + ): + """Insert text work items into the given document.""" + if not self.new_text_work_items: + return + + assert document.home_page_content is not None + layout_index = polarion_html_helper.get_layout_index( + "paragraph", document.rendering_layouts, self.text_work_item_type + ) + html_fragments = polarion_html_helper.ensure_fragments( + document.home_page_content.value + ) + new_content = [] + last_match = -1 + for index, element in enumerate(html_fragments): + if isinstance(element, html.HtmlComment): + continue + + if element.tag == "workitem": + new_content += html_fragments[last_match + 1 : index] + last_match = index + if work_item := self.new_text_work_items.get( + element.get("id") + ): + new_content.append( + html.fromstring( + polarion_html_helper.POLARION_WORK_ITEM_DOCUMENT.format( + pid=work_item.id, + lid=layout_index, + custom_info="", + ) + ) + ) + + new_content += html_fragments[last_match + 1 :] + document.home_page_content.value = "\n".join( + [html.tostring(element).decode("utf-8") for element in new_content] + ) diff --git a/capella2polarion/data_models.py b/capella2polarion/data_models.py index 9ef8153a..b6dcd3c1 100644 --- a/capella2polarion/data_models.py +++ b/capella2polarion/data_models.py @@ -11,6 +11,8 @@ import polarion_rest_api_client as polarion_api +from capella2polarion.converters import text_work_item_provider + class CapellaWorkItem(polarion_api.WorkItem): """A WorkItem class with additional Capella related attributes.""" @@ -68,9 +70,16 @@ class DocumentData: """A class to store data related to a rendered document.""" document: polarion_api.Document - headings: list[polarion_api.WorkItem] = dataclasses.field( - default_factory=list - ) - text_work_items: dict[str, polarion_api.WorkItem] = dataclasses.field( - default_factory=dict - ) + headings: list[polarion_api.WorkItem] + text_work_item_provider: text_work_item_provider.TextWorkItemProvider + + +@dataclasses.dataclass +class DocumentInfo: + """Class for information regarding a document which should be created.""" + + project_id: str | None + module_folder: str + module_name: str + text_work_item_type: str + text_work_item_id_field: str diff --git a/tests/test_documents.py b/tests/test_documents.py index 47f33569..fb55d68f 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -231,7 +231,6 @@ def test_mixed_authority_document( "global_param": "Overwrite global param", }, }, - {}, ) content: list[etree._Element] = html.fromstring( From d847b957f53600e961353923bfaf63709e78aa96 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 3 Sep 2024 11:01:02 +0200 Subject: [PATCH 08/14] ci: fix linting --- capella2polarion/connectors/polarion_worker.py | 8 ++++++-- capella2polarion/converters/link_converter.py | 15 +++++++-------- .../converters/polarion_html_helper.py | 2 +- .../converters/text_work_item_provider.py | 16 ++++++++-------- tests/test_elements.py | 4 ++-- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index a1a348eb..aabb6df5 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -15,7 +15,7 @@ from capella2polarion import data_models from capella2polarion.connectors import polarion_repo -from capella2polarion.converters import data_session, polarion_html_helper +from capella2polarion.converters import data_session logger = logging.getLogger(__name__) @@ -486,7 +486,11 @@ def update_documents( client.work_items.update(headings) client.documents.update(documents) - def _process_document_datas(self, client, document_datas): + def _process_document_datas( + self, + client: polarion_api.ProjectClient, + document_datas: list[data_models.DocumentData], + ): documents = [] headings = [] for document_data in document_datas: diff --git a/capella2polarion/converters/link_converter.py b/capella2polarion/converters/link_converter.py index bf8bb511..a618b087 100644 --- a/capella2polarion/converters/link_converter.py +++ b/capella2polarion/converters/link_converter.py @@ -13,10 +13,13 @@ from capellambse.model import common from capellambse.model import diagram as diag -import capella2polarion.converters.polarion_html_helper from capella2polarion import data_models from capella2polarion.connectors import polarion_repo -from capella2polarion.converters import converter_config, data_session +from capella2polarion.converters import ( + converter_config, + data_session, + polarion_html_helper, +) logger = logging.getLogger(__name__) @@ -346,9 +349,7 @@ def _group_by( def _make_url_list(link_map: dict[str, dict[str, list[str]]]) -> str: urls: list[str] = [] for link_id in sorted(link_map): - url = capella2polarion.converters.polarion_html_helper.POLARION_WORK_ITEM_URL.format( # pylint: disable=line-too-long - pid=link_id - ) + url = polarion_html_helper.POLARION_WORK_ITEM_URL.format(pid=link_id) urls.append(f"
  • {url}
    • ") for key, include_wids in link_map[link_id].items(): _, display_name, _ = key.split(":") @@ -365,9 +366,7 @@ def _sorted_unordered_html_list( ) -> str: urls: list[str] = [] for pid in work_item_ids: - url = capella2polarion.converters.polarion_html_helper.POLARION_WORK_ITEM_URL.format( # pylint: disable=line-too-long - pid=pid - ) + url = polarion_html_helper.POLARION_WORK_ITEM_URL.format(pid=pid) urls.append(f"
  • {url}
    • ") urls.sort() diff --git a/capella2polarion/converters/polarion_html_helper.py b/capella2polarion/converters/polarion_html_helper.py index 9c69cb18..a611c70b 100644 --- a/capella2polarion/converters/polarion_html_helper.py +++ b/capella2polarion/converters/polarion_html_helper.py @@ -114,7 +114,7 @@ def setup_env(self, env: jinja2.Environment): def remove_table_ids( - html_content: str | list[html.HtmlComment], + html_content: str | list[html.HtmlElement], ) -> list[etree._Element]: """Remove the ID field from all tables. diff --git a/capella2polarion/converters/text_work_item_provider.py b/capella2polarion/converters/text_work_item_provider.py index cb74ae4f..c1b8582b 100644 --- a/capella2polarion/converters/text_work_item_provider.py +++ b/capella2polarion/converters/text_work_item_provider.py @@ -4,7 +4,7 @@ import polarion_rest_api_client as polarion_api from lxml import html -from capella2polarion.converters import polarion_html_helper +from capella2polarion.converters import polarion_html_helper as html_helper class TextWorkItemProvider: @@ -12,8 +12,8 @@ class TextWorkItemProvider: def __init__( self, - text_work_item_id_field: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD, - text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE, + text_work_item_id_field: str = html_helper.TEXT_WORK_ITEM_ID_FIELD, + text_work_item_type: str = html_helper.TEXT_WORK_ITEM_TYPE, existing_text_work_items: list[polarion_api.WorkItem] | None = None, ): self.old_text_work_items: dict[str, polarion_api.WorkItem] = {} @@ -40,9 +40,9 @@ def generate_text_work_items( work_item_id_filter: list[str] | None = None, ): """Generate text work items from the provided html.""" - content = polarion_html_helper.ensure_fragments(content) + content = html_helper.ensure_fragments(content) for element in content: - if element.tag != polarion_html_helper.WORK_ITEM_TAG: + if element.tag != html_helper.WORK_ITEM_TAG: continue if not (text_id := element.get("id")): @@ -88,10 +88,10 @@ def insert_text_work_items( return assert document.home_page_content is not None - layout_index = polarion_html_helper.get_layout_index( + layout_index = html_helper.get_layout_index( "paragraph", document.rendering_layouts, self.text_work_item_type ) - html_fragments = polarion_html_helper.ensure_fragments( + html_fragments = html_helper.ensure_fragments( document.home_page_content.value ) new_content = [] @@ -108,7 +108,7 @@ def insert_text_work_items( ): new_content.append( html.fromstring( - polarion_html_helper.POLARION_WORK_ITEM_DOCUMENT.format( + html_helper.POLARION_WORK_ITEM_DOCUMENT.format( pid=work_item.id, lid=layout_index, custom_info="", diff --git a/tests/test_elements.py b/tests/test_elements.py index 95f94dd2..1c8e88eb 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -795,7 +795,7 @@ def test_update_work_items( assert base_object.pw.project_client.work_items.update.call_count == 1 assert base_object.pw.project_client.work_items.get.call_count == 1 assert ( - base_object.pw.project_client.work_items.attachments.get_all.call_count + base_object.pw.project_client.work_items.attachments.get_all.call_count # pylint: disable=line-too-long == 0 ) work_item = base_object.pw.project_client.work_items.update.call_args[ @@ -1017,7 +1017,7 @@ def test_update_links(base_object: BaseObjectContainer): base_object.mc.converter_session ) links = ( - base_object.pw.project_client.work_items.links.get_all.call_args_list + base_object.pw.project_client.work_items.links.get_all.call_args_list # pylint: disable=line-too-long ) assert ( base_object.pw.project_client.work_items.links.get_all.call_count From 5af17194b9bf8dd58ac5d02f3070d94ac9915731 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 3 Sep 2024 11:27:37 +0200 Subject: [PATCH 09/14] fix: update of text work items --- .../converters/text_work_item_provider.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/capella2polarion/converters/text_work_item_provider.py b/capella2polarion/converters/text_work_item_provider.py index c1b8582b..ae9a7455 100644 --- a/capella2polarion/converters/text_work_item_provider.py +++ b/capella2polarion/converters/text_work_item_provider.py @@ -48,11 +48,13 @@ def generate_text_work_items( if not (text_id := element.get("id")): raise ValueError("All work items must have an ID in template") - if ( - work_item_id_filter is None or text_id in work_item_id_filter - ) and text_id in self.old_text_work_items: - work_item = self.old_text_work_items[text_id] - else: + if not ( + (work_item := self.old_text_work_items.get(text_id)) + and ( + work_item_id_filter is None + or work_item.id in work_item_id_filter + ) + ): work_item = polarion_api.WorkItem( type=self.text_work_item_type, title="", From dfe28ec7ee5a7de89e9cdfeb2e8624fd2febe534 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 3 Sep 2024 12:54:00 +0200 Subject: [PATCH 10/14] test: add tests --- capella2polarion/__main__.py | 2 +- .../connectors/polarion_worker.py | 2 +- .../converters/polarion_html_helper.py | 38 ---- .../converters/text_work_item_provider.py | 1 + tests/conftest.py | 4 + .../data/documents/sections/section1.html.j2 | 1 + .../data/documents/sections/section2.html.j2 | 1 + .../templates/document_work_items.html.j2 | 10 + tests/test_cli.py | 16 +- tests/test_documents.py | 159 +++++++++++++++- tests/test_polarion_worker_documents.py | 178 ++++++++++++++++++ 11 files changed, 355 insertions(+), 57 deletions(-) create mode 100644 tests/data/documents/templates/document_work_items.html.j2 create mode 100644 tests/test_polarion_worker_documents.py diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 437588b3..a9e93ea6 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -169,7 +169,7 @@ def render_documents( projects_document_data = renderer.render_documents(configs, documents) for project, project_data in projects_document_data.items(): - polarion_worker.post_documents(project_data.new_docs, project) + polarion_worker.create_documents(project_data.new_docs, project) polarion_worker.update_documents(project_data.updated_docs, project) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index aabb6df5..48f80580 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -461,7 +461,7 @@ def compare_and_update_work_items( if uuid in self.polarion_data_repo and data.work_item is not None: self.compare_and_update_work_item(data) - def post_documents( + def create_documents( self, document_datas: list[data_models.DocumentData], document_project: str | None = None, diff --git a/capella2polarion/converters/polarion_html_helper.py b/capella2polarion/converters/polarion_html_helper.py index a611c70b..0a684a74 100644 --- a/capella2polarion/converters/polarion_html_helper.py +++ b/capella2polarion/converters/polarion_html_helper.py @@ -165,44 +165,6 @@ def extract_work_items( return work_items -def insert_text_work_items( - document: polarion_api.Document, - text_work_items: dict[str, polarion_api.WorkItem], - text_work_item_type: str, -): - """Insert text work items into the given document.""" - if not text_work_items: - return - - assert document.home_page_content is not None - layout_index = get_layout_index( - "paragraph", document.rendering_layouts, text_work_item_type - ) - html_fragments = ensure_fragments(document.home_page_content.value) - new_content = [] - last_match = -1 - for index, element in enumerate(html_fragments): - if isinstance(element, html.HtmlComment): - continue - - if element.tag == "workitem": - new_content += html_fragments[last_match + 1 : index] - last_match = index - if work_item := text_work_items.get(element.get("id")): - new_content.append( - html.fromstring( - POLARION_WORK_ITEM_DOCUMENT.format( - pid=work_item.id, lid=layout_index, custom_info="" - ) - ) - ) - - new_content += html_fragments[last_match + 1 :] - document.home_page_content.value = "\n".join( - [html.tostring(element).decode("utf-8") for element in new_content] - ) - - def get_layout_index( default_layouter: str, rendering_layouts: list[polarion_api.RenderingLayout], diff --git a/capella2polarion/converters/text_work_item_provider.py b/capella2polarion/converters/text_work_item_provider.py index ae9a7455..4bae1845 100644 --- a/capella2polarion/converters/text_work_item_provider.py +++ b/capella2polarion/converters/text_work_item_provider.py @@ -90,6 +90,7 @@ def insert_text_work_items( return assert document.home_page_content is not None + assert document.rendering_layouts is not None layout_index = html_helper.get_layout_index( "paragraph", document.rendering_layouts, self.text_work_item_type ) diff --git a/tests/conftest.py b/tests/conftest.py index 2d5afc09..296cd368 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -186,3 +186,7 @@ def empty_polarion_worker(monkeypatch: pytest.MonkeyPatch): delete_work_items=True, ) yield polarion_worker.CapellaPolarionWorker(polarion_params) + + +DOCUMENT_TEMPLATES = TEST_DOCUMENT_ROOT / "templates" +DOCUMENT_TEXT_WORK_ITEMS = "document_work_items.html.j2" diff --git a/tests/data/documents/sections/section1.html.j2 b/tests/data/documents/sections/section1.html.j2 index 575220b9..13b1afb7 100644 --- a/tests/data/documents/sections/section1.html.j2 +++ b/tests/data/documents/sections/section1.html.j2 @@ -6,3 +6,4 @@ {{ heading(3, "New Heading", session) }}

    {{ global_param }}

    {{ local_param }}

    +TestContent diff --git a/tests/data/documents/sections/section2.html.j2 b/tests/data/documents/sections/section2.html.j2 index ec7a3fb7..0e0630a0 100644 --- a/tests/data/documents/sections/section2.html.j2 +++ b/tests/data/documents/sections/section2.html.j2 @@ -6,3 +6,4 @@ {{ heading(3, "Keep Heading", session) }}

    Overwritten: {{ global_param }}

    {{ local_param }}

    +TestContent diff --git a/tests/data/documents/templates/document_work_items.html.j2 b/tests/data/documents/templates/document_work_items.html.j2 new file mode 100644 index 00000000..aecd905a --- /dev/null +++ b/tests/data/documents/templates/document_work_items.html.j2 @@ -0,0 +1,10 @@ + + +This is Text in a text workitem + +Text +
    12
    +     diff --git a/tests/test_cli.py b/tests/test_cli.py index b102592a..7894f405 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -117,11 +117,11 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): "get_document", mock_get_document, ) - mock_post_documents = mock.MagicMock() + mock_create_documents = mock.MagicMock() monkeypatch.setattr( polarion_worker.CapellaPolarionWorker, - "post_documents", - mock_post_documents, + "create_documents", + mock_create_documents, ) mock_update_documents = mock.MagicMock() monkeypatch.setattr( @@ -161,11 +161,11 @@ def test_render_documents(monkeypatch: pytest.MonkeyPatch): "TestProject", ] - assert mock_post_documents.call_count == 2 - assert len(mock_post_documents.call_args_list[0].args[0]) == 1 - assert len(mock_post_documents.call_args_list[1].args[0]) == 1 - assert mock_post_documents.call_args_list[0].args[1] is None - assert mock_post_documents.call_args_list[1].args[1] == "TestProject" + assert mock_create_documents.call_count == 2 + assert len(mock_create_documents.call_args_list[0].args[0]) == 1 + assert len(mock_create_documents.call_args_list[1].args[0]) == 1 + assert mock_create_documents.call_args_list[0].args[1] is None + assert mock_create_documents.call_args_list[1].args[1] == "TestProject" assert mock_update_documents.call_count == 2 assert len(mock_update_documents.call_args_list[0].args[0]) == 1 diff --git a/tests/test_documents.py b/tests/test_documents.py index fb55d68f..1faa13ce 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -7,8 +7,17 @@ from capella2polarion import data_models as dm from capella2polarion.connectors import polarion_worker -from capella2polarion.converters import document_config, document_renderer -from tests.conftest import TEST_COMBINED_DOCUMENT_CONFIG, TEST_DOCUMENT_ROOT +from capella2polarion.converters import ( + document_config, + document_renderer, + text_work_item_provider, +) +from tests.conftest import ( + DOCUMENT_TEMPLATES, + DOCUMENT_TEXT_WORK_ITEMS, + TEST_COMBINED_DOCUMENT_CONFIG, + TEST_DOCUMENT_ROOT, +) CLASSES_TEMPLATE = "test-classes.html.j2" JUPYTER_TEMPLATE_FOLDER = "jupyter-notebooks/document_templates" @@ -231,30 +240,162 @@ def test_mixed_authority_document( "global_param": "Overwrite global param", }, }, + text_work_item_provider=text_work_item_provider.TextWorkItemProvider( + "MyField", + "MyType", + [ + polarion_api.WorkItem( + id="EXISTING", additional_attributes={"MyField": "id1"} + ) + ], + ), ) content: list[etree._Element] = html.fromstring( document_data.document.home_page_content.value ) - assert len(content) == 15 + assert len(document_data.text_work_item_provider.new_text_work_items) == 2 + assert ( + document_data.text_work_item_provider.new_text_work_items["id1"].id + is None + ) + assert ( + document_data.text_work_item_provider.new_text_work_items["id2"].id + is None + ) + assert len(content) == 17 assert [c.tag for c in content[:3]] == ["h1", "p", "p"] assert (c4 := content[4]).tag == "h3" and c4.text == "New Heading" assert content[5].text == "Global Test" assert content[6].text == "Local Test section 1" - assert content[8].text == "This will be kept." - assert content[10].get("id") == ( + assert content[9].text == "This will be kept." + assert content[11].get("id") == ( "polarion_wiki macro name=module-workitem;params=id=ATSY-18305" ) - assert content[10].tag == "h3" - assert content[11].text == "Overwritten: Overwrite global param" - assert content[12].text == "Local Test section 2" - assert content[14].text == "Some postfix stuff" + assert content[11].tag == "h3" + assert content[12].text == "Overwritten: Overwrite global param" + assert content[13].text == "Local Test section 2" + assert content[16].text == "Some postfix stuff" assert len(document_data.headings) == 1 assert document_data.headings[0].id == "ATSY-18305" assert document_data.headings[0].title == "Keep Heading" +def test_create_full_authority_document_text_work_items( + empty_polarion_worker: polarion_worker.CapellaPolarionWorker, + model: capellambse.MelodyModel, +): + renderer = document_renderer.DocumentRenderer( + empty_polarion_worker.polarion_data_repo, model + ) + + document_data = renderer.render_document( + DOCUMENT_TEMPLATES, + DOCUMENT_TEXT_WORK_ITEMS, + "_default", + "TEST-DOC", + text_work_item_provider=text_work_item_provider.TextWorkItemProvider( + "MyField", + "MyType", + ), + ) + + assert len(document_data.text_work_item_provider.new_text_work_items) == 2 + assert ( + document_data.text_work_item_provider.new_text_work_items["id1"].id + is None + ) + assert ( + document_data.text_work_item_provider.new_text_work_items["id1"].type + == "MyType" + ) + assert ( + document_data.text_work_item_provider.new_text_work_items[ + "id1" + ].additional_attributes["MyField"] + == "id1" + ) + assert ( + document_data.text_work_item_provider.new_text_work_items["id2"].id + is None + ) + assert ( + document_data.text_work_item_provider.new_text_work_items["id2"].type + == "MyType" + ) + assert ( + document_data.text_work_item_provider.new_text_work_items[ + "id2" + ].additional_attributes["MyField"] + == "id2" + ) + + +def test_update_full_authority_document_text_work_items( + empty_polarion_worker: polarion_worker.CapellaPolarionWorker, + model: capellambse.MelodyModel, +): + renderer = document_renderer.DocumentRenderer( + empty_polarion_worker.polarion_data_repo, model + ) + old_doc = polarion_api.Document( + module_folder="_default", + module_name="TEST-DOC", + home_page_content=polarion_api.TextContent( + type="text/html", + value="", + ), + ) + + document_data = renderer.render_document( + DOCUMENT_TEMPLATES, + DOCUMENT_TEXT_WORK_ITEMS, + "_default", + "TEST-DOC", + document=old_doc, + text_work_item_provider=text_work_item_provider.TextWorkItemProvider( + "MyField", + "MyType", + [ + polarion_api.WorkItem( + id="EXISTING", additional_attributes={"MyField": "id1"} + ) + ], + ), + ) + + assert len(document_data.text_work_item_provider.new_text_work_items) == 2 + assert ( + document_data.text_work_item_provider.new_text_work_items["id1"].id + == "EXISTING" + ) + assert ( + document_data.text_work_item_provider.new_text_work_items["id1"].type + is None + ) + assert ( + document_data.text_work_item_provider.new_text_work_items[ + "id1" + ].additional_attributes["MyField"] + == "id1" + ) + assert ( + document_data.text_work_item_provider.new_text_work_items["id2"].id + is None + ) + assert ( + document_data.text_work_item_provider.new_text_work_items["id2"].type + == "MyType" + ) + assert ( + document_data.text_work_item_provider.new_text_work_items[ + "id2" + ].additional_attributes["MyField"] + == "id2" + ) + + def test_render_all_documents_partially_successfully( empty_polarion_worker: polarion_worker.CapellaPolarionWorker, model: capellambse.MelodyModel, diff --git a/tests/test_polarion_worker_documents.py b/tests/test_polarion_worker_documents.py new file mode 100644 index 00000000..561c9fc3 --- /dev/null +++ b/tests/test_polarion_worker_documents.py @@ -0,0 +1,178 @@ +# Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import polarion_rest_api_client as polarion_api + +from capella2polarion import data_models +from capella2polarion.connectors import polarion_worker +from capella2polarion.converters import text_work_item_provider + +from .conftest import DOCUMENT_TEMPLATES, DOCUMENT_TEXT_WORK_ITEMS + + +def _set_work_item_id(work_items: list[polarion_api.WorkItem]): + for index, work_item in enumerate(work_items): + work_item.id = f"id{index}" + + +def test_update_document( + empty_polarion_worker: polarion_worker.CapellaPolarionWorker, +): + path = DOCUMENT_TEMPLATES / DOCUMENT_TEXT_WORK_ITEMS + document = polarion_api.Document( + module_folder="_default", + module_name="TEST-DOC", + rendering_layouts=[], + home_page_content=polarion_api.TextContent( + type="text/html", + value=path.read_text("utf-8"), + ), + ) + document_data = data_models.DocumentData( + document, + [], + text_work_item_provider.TextWorkItemProvider( + "MyField", + "MyType", + [ + polarion_api.WorkItem( + id="EXISTING", additional_attributes={"MyField": "id1"} + ), + ], + ), + ) + document_data.text_work_item_provider.generate_text_work_items( + document.home_page_content.value + ) + empty_polarion_worker.project_client.work_items.create.side_effect = ( + _set_work_item_id + ) + + empty_polarion_worker.update_documents([document_data]) + + assert document.home_page_content.value.endswith( + '
    \n' + '
    ' + ) + assert ( + empty_polarion_worker.project_client.documents.update.call_count == 1 + ) + assert ( + empty_polarion_worker.project_client.documents.update.call_args.args[0] + == [document] + ) + assert ( + empty_polarion_worker.project_client.work_items.create.call_count == 1 + ) + assert ( + len( + empty_polarion_worker.project_client.work_items.create.call_args.args[ + 0 + ] + ) + == 1 + ) + assert ( + empty_polarion_worker.project_client.work_items.update.call_count == 2 + ) + assert ( + len( + empty_polarion_worker.project_client.work_items.update.call_args_list[ + 0 + ].args[ + 0 + ] + ) + == 1 + ) + assert ( + len( + empty_polarion_worker.project_client.work_items.update.call_args_list[ + 1 + ].args[ + 0 + ] + ) + == 0 + ) + + +def test_create_document( + empty_polarion_worker: polarion_worker.CapellaPolarionWorker, +): + path = DOCUMENT_TEMPLATES / DOCUMENT_TEXT_WORK_ITEMS + document = polarion_api.Document( + module_folder="_default", + module_name="TEST-DOC", + rendering_layouts=[], + home_page_content=polarion_api.TextContent( + type="text/html", + value=path.read_text("utf-8"), + ), + ) + document_data = data_models.DocumentData( + document, + [], + text_work_item_provider.TextWorkItemProvider( + "MyField", + "MyType", + ), + ) + document_data.text_work_item_provider.generate_text_work_items( + document.home_page_content.value + ) + empty_polarion_worker.project_client.work_items.create.side_effect = ( + _set_work_item_id + ) + + empty_polarion_worker.update_documents([document_data]) + + assert document.home_page_content.value.endswith( + '
    \n' + '
    ' + ) + assert ( + empty_polarion_worker.project_client.documents.update.call_count == 1 + ) + assert ( + empty_polarion_worker.project_client.documents.update.call_args.args[0] + == [document] + ) + assert ( + empty_polarion_worker.project_client.work_items.create.call_count == 1 + ) + assert ( + len( + empty_polarion_worker.project_client.work_items.create.call_args.args[ + 0 + ] + ) + == 2 + ) + assert ( + empty_polarion_worker.project_client.work_items.update.call_count == 2 + ) + assert ( + len( + empty_polarion_worker.project_client.work_items.update.call_args_list[ + 0 + ].args[ + 0 + ] + ) + == 0 + ) + assert ( + len( + empty_polarion_worker.project_client.work_items.update.call_args_list[ + 1 + ].args[ + 0 + ] + ) + == 0 + ) From fc7f4e74f4e7cad10866983b6761f2d2a70380da Mon Sep 17 00:00:00 2001 From: micha91 Date: Wed, 4 Sep 2024 07:13:29 +0200 Subject: [PATCH 11/14] chore: Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ernst Würger --- .../connectors/polarion_worker.py | 47 ++++++++++++++----- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 48f80580..a63e1e34 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -123,6 +123,7 @@ def load_polarion_work_item_map(self): work_items = self.project_client.work_items.get_all( "HAS_VALUE:uuid_capella", fields={"workitems": "id,uuid_capella,checksum,status,type"}, + work_item_cls=data_models.CapellaWorkItem, ) self.polarion_data_repo.update_work_items(work_items) @@ -141,11 +142,13 @@ def delete_orphaned_work_items( if work_item.status != "deleted" } uuids: set[str] = existing_work_items - set(converter_session) - work_items = [ - self.polarion_data_repo.get_work_item_by_capella_uuid(uuid) - for uuid in uuids - ] - if work_items: + work_items: list[data_models.CapellaWorkitem] = [] + for uuid in uuids: + if wi := self.polarion_data_repo.get_work_item_by_capella_uuid( + uuid + ): + logger.info("Delete work item %r...", wi.id) + work_items.append(wi) try: self.project_client.work_items.delete(work_items) self.polarion_data_repo.remove_work_items_by_capella_uuid( @@ -214,7 +217,9 @@ def compare_and_update_work_item( work_item_changed = new_work_item_check_sum != old_work_item_check_sum try: if work_item_changed or self.force_update: - old = self.project_client.work_items.get(old.id) + old = self.project_client.work_items.get( + old.id, work_item_cls=data_models.CapellaWorkItem + ) if old.attachments: old_attachments = ( self.project_client.work_items.attachments.get_all( @@ -466,7 +471,13 @@ def create_documents( document_datas: list[data_models.DocumentData], document_project: str | None = None, ): - """Create new documents.""" + """Create new documents. + + Notes + ----- + If the ``document_project`` is ``None`` the default client is + taken. + """ client = self._get_client(document_project) documents, _ = self._process_document_datas(client, document_datas) @@ -477,7 +488,13 @@ def update_documents( document_datas: list[data_models.DocumentData], document_project: str | None = None, ): - """Update existing documents.""" + """Update existing documents. + + Notes + ----- + If the ``document_project`` is ``None`` the default client is + taken. + """ client = self._get_client(document_project) documents, headings = self._process_document_datas( client, document_datas @@ -491,10 +508,10 @@ def _process_document_datas( client: polarion_api.ProjectClient, document_datas: list[data_models.DocumentData], ): - documents = [] - headings = [] + documents: list[polarion_api.Document] = [] + headings: list[polarion_api.WorkItem] = [] for document_data in document_datas: - headings += document_data.headings + headings.extend(document_data.headings) documents.append(document_data.document) if document_data.text_work_item_provider.new_text_work_items: self._create_and_update_text_work_items( @@ -509,7 +526,13 @@ def _process_document_datas( def get_document( self, space: str, name: str, document_project: str | None = None ) -> polarion_api.Document | None: - """Get a document from polarion and return None if not found.""" + """Get a document from polarion and return None if not found. + + Notes + ----- + If the ``document_project`` is ``None`` the default client is + taken. + """ client = self._get_client(document_project) try: return client.documents.get( From 22a5a59176f5a6b6260232a3820b210ee6d44caf Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Wed, 4 Sep 2024 08:59:30 +0200 Subject: [PATCH 12/14] chore: Additional changes from review --- capella2polarion/connectors/polarion_repo.py | 13 +++ .../connectors/polarion_worker.py | 10 +- .../converters/document_renderer.py | 10 +- .../converters/polarion_html_helper.py | 27 +++-- .../converters/text_work_item_provider.py | 2 +- tests/test_documents.py | 7 +- tests/test_polarion_worker_documents.py | 106 +++--------------- 7 files changed, 56 insertions(+), 119 deletions(-) diff --git a/capella2polarion/connectors/polarion_repo.py b/capella2polarion/connectors/polarion_repo.py index 7acd47d0..f40f70ca 100644 --- a/capella2polarion/connectors/polarion_repo.py +++ b/capella2polarion/connectors/polarion_repo.py @@ -6,6 +6,7 @@ import collections.abc as cabc import bidict +import polarion_rest_api_client as polarion_api from capella2polarion import data_models @@ -114,3 +115,15 @@ def remove_work_items_by_capella_uuid(self, uuids: cabc.Iterable[str]): for uuid in uuids: del self._work_items[uuid] del self._id_mapping[uuid] + + +DocumentRepository = dict[ + tuple[str | None, str, str], + tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], +] +"""A dict providing a mapping for documents and their text workitems. + +It has (project, space, name) of the document as key and (document, +workitems) as value. The project can be None and the None value means +that the document is in the same project as the model sync work items. +""" diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index a63e1e34..dd84dfb6 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -33,6 +33,7 @@ "EXISTS (SELECT rel1.* FROM POLARION.REL_MODULE_WORKITEM rel1 WHERE " "rel1.FK_URI_MODULE = doc.C_URI AND rel1.FK_URI_WORKITEM = item.C_URI))" ) +"""An SQL query to get work items which are inserted in a given document.""" class PolarionWorkerParams: @@ -142,7 +143,7 @@ def delete_orphaned_work_items( if work_item.status != "deleted" } uuids: set[str] = existing_work_items - set(converter_session) - work_items: list[data_models.CapellaWorkitem] = [] + work_items: list[data_models.CapellaWorkItem] = [] for uuid in uuids: if wi := self.polarion_data_repo.get_work_item_by_capella_uuid( uuid @@ -527,7 +528,7 @@ def get_document( self, space: str, name: str, document_project: str | None = None ) -> polarion_api.Document | None: """Get a document from polarion and return None if not found. - + Notes ----- If the ``document_project`` is ``None`` the default client is @@ -546,10 +547,7 @@ def get_document( def load_polarion_documents( self, document_infos: t.Iterable[data_models.DocumentInfo], - ) -> dict[ - tuple[str | None, str, str], - tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], - ]: + ) -> polarion_repo.DocumentRepository: """Load the documents referenced and text work items from Polarion.""" return { (di.project_id, di.module_folder, di.module_name): ( diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index a0c1766a..58e7af59 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -70,10 +70,7 @@ def __init__( self.overwrite_heading_numbering = overwrite_heading_numbering self.overwrite_layouts = overwrite_layouts self.projects: dict[str | None, ProjectData] = {} - self.existing_documents: dict[ - tuple[str | None, str, str], - tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], - ] = {} + self.existing_documents: polarion_repo.DocumentRepository = {} def setup_env(self, env: jinja2.Environment): """Add globals and filters to the environment.""" @@ -354,10 +351,7 @@ def _get_and_customize_doc( def render_documents( self, configs: document_config.DocumentConfigs, - existing_documents: dict[ - tuple[str | None, str, str], - tuple[polarion_api.Document | None, list[polarion_api.WorkItem]], - ], + existing_documents: polarion_repo.DocumentRepository, ) -> dict[str | None, ProjectData]: """Render all documents defined in the given config. diff --git a/capella2polarion/converters/polarion_html_helper.py b/capella2polarion/converters/polarion_html_helper.py index 0a684a74..2cc20ac7 100644 --- a/capella2polarion/converters/polarion_html_helper.py +++ b/capella2polarion/converters/polarion_html_helper.py @@ -10,7 +10,7 @@ import jinja2 import polarion_rest_api_client as polarion_api from capellambse import helpers as chelpers -from lxml import etree, html +from lxml import html wi_id_prefix = "polarion_wiki macro name=module-workitem;params=id=" h_regex = re.compile("h[0-9]") @@ -114,8 +114,8 @@ def setup_env(self, env: jinja2.Environment): def remove_table_ids( - html_content: str | list[html.HtmlElement], -) -> list[etree._Element]: + html_content: str | list[html.HtmlElement | str], +) -> list[html.HtmlElement | str]: """Remove the ID field from all tables. This is necessary due to a bug in Polarion where Polarion does not @@ -126,6 +126,9 @@ def remove_table_ids( html_fragments = ensure_fragments(html_content) for element in html_fragments: + if not isinstance(element, html.HtmlElement): + continue + if element.tag == "table": element.attrib.pop("id", None) @@ -133,36 +136,38 @@ def remove_table_ids( def ensure_fragments( - html_content: str | list[html.HtmlElement], -) -> list[html.HtmlElement]: + html_content: str | list[html.HtmlElement | str], +) -> list[html.HtmlElement | str]: """Convert string to html elements.""" if isinstance(html_content, str): return html.fragments_fromstring(html_content) return html_content -def extract_headings(html_content: str | list[html.HtmlElement]) -> list[str]: +def extract_headings( + html_content: str | list[html.HtmlElement | str], +) -> list[str]: """Return a list of work item IDs for all headings in the given content.""" return extract_work_items(html_content, h_regex) def extract_work_items( - html_content: str | list[html.HtmlElement], + html_content: str | list[html.HtmlElement | str], tag_regex: re.Pattern | None = None, ) -> list[str]: """Return a list of work item IDs for work items in the given content.""" - work_items = [] + work_item_ids: list[str] = [] html_fragments = ensure_fragments(html_content) for element in html_fragments: - if isinstance(element, html.HtmlComment): + if not isinstance(element, html.HtmlElement): continue if (tag_regex is not None and tag_regex.fullmatch(element.tag)) or ( tag_regex is None and element.tag == "div" ): if matches := wi_id_regex.match(element.get("id")): - work_items.append(matches.group(1)) - return work_items + work_item_ids.append(matches.group(1)) + return work_item_ids def get_layout_index( diff --git a/capella2polarion/converters/text_work_item_provider.py b/capella2polarion/converters/text_work_item_provider.py index 4bae1845..a75e022a 100644 --- a/capella2polarion/converters/text_work_item_provider.py +++ b/capella2polarion/converters/text_work_item_provider.py @@ -100,7 +100,7 @@ def insert_text_work_items( new_content = [] last_match = -1 for index, element in enumerate(html_fragments): - if isinstance(element, html.HtmlComment): + if not isinstance(element, html.HtmlElement): continue if element.tag == "workitem": diff --git a/tests/test_documents.py b/tests/test_documents.py index 1faa13ce..69e046f5 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -6,7 +6,7 @@ from lxml import etree, html from capella2polarion import data_models as dm -from capella2polarion.connectors import polarion_worker +from capella2polarion.connectors import polarion_repo, polarion_worker from capella2polarion.converters import ( document_config, document_renderer, @@ -28,10 +28,7 @@ MIXED_AUTHORITY_DOCUMENT = TEST_DOCUMENT_ROOT / "mixed_authority_doc.html" -def existing_documents() -> dict[ - tuple[str | None, str, str], - tuple[polarion_api.Document, list[polarion_api.WorkItem]], -]: +def existing_documents() -> polarion_repo.DocumentRepository: return { (None, "_default", "id123"): ( polarion_api.Document( diff --git a/tests/test_polarion_worker_documents.py b/tests/test_polarion_worker_documents.py index 561c9fc3..949ca9f0 100644 --- a/tests/test_polarion_worker_documents.py +++ b/tests/test_polarion_worker_documents.py @@ -44,9 +44,8 @@ def test_update_document( document_data.text_work_item_provider.generate_text_work_items( document.home_page_content.value ) - empty_polarion_worker.project_client.work_items.create.side_effect = ( - _set_work_item_id - ) + client = empty_polarion_worker.project_client + client.work_items.create.side_effect = _set_work_item_id empty_polarion_worker.update_documents([document_data]) @@ -56,47 +55,13 @@ def test_update_document( '
    ' ) - assert ( - empty_polarion_worker.project_client.documents.update.call_count == 1 - ) - assert ( - empty_polarion_worker.project_client.documents.update.call_args.args[0] - == [document] - ) - assert ( - empty_polarion_worker.project_client.work_items.create.call_count == 1 - ) - assert ( - len( - empty_polarion_worker.project_client.work_items.create.call_args.args[ - 0 - ] - ) - == 1 - ) - assert ( - empty_polarion_worker.project_client.work_items.update.call_count == 2 - ) - assert ( - len( - empty_polarion_worker.project_client.work_items.update.call_args_list[ - 0 - ].args[ - 0 - ] - ) - == 1 - ) - assert ( - len( - empty_polarion_worker.project_client.work_items.update.call_args_list[ - 1 - ].args[ - 0 - ] - ) - == 0 - ) + assert client.documents.update.call_count == 1 + assert client.documents.update.call_args.args[0] == [document] + assert client.work_items.create.call_count == 1 + assert len(client.work_items.create.call_args.args[0]) == 1 + assert client.work_items.update.call_count == 2 + assert len(client.work_items.update.call_args_list[0].args[0]) == 1 + assert len(client.work_items.update.call_args_list[1].args[0]) == 0 def test_create_document( @@ -123,9 +88,8 @@ def test_create_document( document_data.text_work_item_provider.generate_text_work_items( document.home_page_content.value ) - empty_polarion_worker.project_client.work_items.create.side_effect = ( - _set_work_item_id - ) + client = empty_polarion_worker.project_client + client.work_items.create.side_effect = _set_work_item_id empty_polarion_worker.update_documents([document_data]) @@ -135,44 +99,10 @@ def test_create_document( '
    ' ) - assert ( - empty_polarion_worker.project_client.documents.update.call_count == 1 - ) - assert ( - empty_polarion_worker.project_client.documents.update.call_args.args[0] - == [document] - ) - assert ( - empty_polarion_worker.project_client.work_items.create.call_count == 1 - ) - assert ( - len( - empty_polarion_worker.project_client.work_items.create.call_args.args[ - 0 - ] - ) - == 2 - ) - assert ( - empty_polarion_worker.project_client.work_items.update.call_count == 2 - ) - assert ( - len( - empty_polarion_worker.project_client.work_items.update.call_args_list[ - 0 - ].args[ - 0 - ] - ) - == 0 - ) - assert ( - len( - empty_polarion_worker.project_client.work_items.update.call_args_list[ - 1 - ].args[ - 0 - ] - ) - == 0 - ) + assert client.documents.update.call_count == 1 + assert client.documents.update.call_args.args[0] == [document] + assert client.work_items.create.call_count == 1 + assert len(client.work_items.create.call_args.args[0]) == 2 + assert client.work_items.update.call_count == 2 + assert len(client.work_items.update.call_args_list[0].args[0]) == 0 + assert len(client.work_items.update.call_args_list[1].args[0]) == 0 From 1b775ba35dc5e9c388d7d1c3664abd666fee8791 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Wed, 4 Sep 2024 09:54:19 +0200 Subject: [PATCH 13/14] test: add additional tests for new features --- tests/data/documents/mixed_config.yaml | 2 ++ tests/test_documents.py | 4 +++ tests/test_polarion_worker_documents.py | 34 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/tests/data/documents/mixed_config.yaml b/tests/data/documents/mixed_config.yaml index fb346aa8..f3ca0586 100644 --- a/tests/data/documents/mixed_config.yaml +++ b/tests/data/documents/mixed_config.yaml @@ -25,6 +25,8 @@ mixed_authority: section1: test-icd.html.j2 section2: test-icd.html.j2 heading_numbering: True + text_work_item_type: myType + text_work_item_id_field: myId work_item_layouts: componentExchange: fields_at_start: diff --git a/tests/test_documents.py b/tests/test_documents.py index 69e046f5..5069b46d 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -523,6 +523,10 @@ def test_mixed_authority_document_config(): } assert conf.mixed_authority[1].project_id is None assert conf.mixed_authority[1].status_allow_list is None + assert conf.mixed_authority[0].text_work_item_type == "text" + assert conf.mixed_authority[0].text_work_item_id_field == "__C2P__id" + assert conf.mixed_authority[1].text_work_item_type == "myType" + assert conf.mixed_authority[1].text_work_item_id_field == "myId" def test_combined_config(): diff --git a/tests/test_polarion_worker_documents.py b/tests/test_polarion_worker_documents.py index 949ca9f0..714847bb 100644 --- a/tests/test_polarion_worker_documents.py +++ b/tests/test_polarion_worker_documents.py @@ -1,5 +1,6 @@ # Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 +from unittest import mock import polarion_rest_api_client as polarion_api @@ -106,3 +107,36 @@ def test_create_document( assert client.work_items.update.call_count == 2 assert len(client.work_items.update.call_args_list[0].args[0]) == 0 assert len(client.work_items.update.call_args_list[1].args[0]) == 0 + + +def test_use_correct_client( + empty_polarion_worker: polarion_worker.CapellaPolarionWorker, +): + empty_polarion_worker.project_client = mock.MagicMock() + document = polarion_api.Document( + module_folder="_default", + module_name="TEST-DOC-A", + rendering_layouts=[], + home_page_content=polarion_api.TextContent( + type="text/html", + value="", + ), + ) + + document_data = data_models.DocumentData( + document, + [], + text_work_item_provider.TextWorkItemProvider(), + ) + + empty_polarion_worker.create_documents([document_data], "OtherProject") + empty_polarion_worker.update_documents([document_data], "OtherProject") + + assert len(empty_polarion_worker.project_client.method_calls) == 0 + assert len(empty_polarion_worker._additional_clients) == 1 + assert ( + client := empty_polarion_worker._additional_clients.get("OtherProject") + ) + assert client.documents.update.call_count == 1 + assert client.documents.create.call_count == 1 + assert client.work_items.update.call_count == 1 From 6abc00a2e75ce6fefab200b7f87854e6dae6b0bc Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Wed, 4 Sep 2024 12:28:22 +0200 Subject: [PATCH 14/14] fix: we should only delete the work_items once from the mapping, otherwise all runs with more than one work item to be deleted will fail --- capella2polarion/connectors/polarion_worker.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index dd84dfb6..e6b22371 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -152,13 +152,12 @@ def delete_orphaned_work_items( work_items.append(wi) try: self.project_client.work_items.delete(work_items) - self.polarion_data_repo.remove_work_items_by_capella_uuid( - uuids - ) except polarion_api.PolarionApiException as error: logger.error("Deleting work items failed. %s", error.args[0]) raise error + self.polarion_data_repo.remove_work_items_by_capella_uuid(uuids) + def create_missing_work_items( self, converter_session: data_session.ConverterSession ) -> None: