From 49449eb2b145fbff62ba283a313654ad4cf8165f Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 15 Aug 2024 11:37:14 +0200 Subject: [PATCH 1/7] chore: raise all polarion API exceptions --- capella2polarion/connectors/polarion_worker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index b61ca1ec..4e4c172d 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -123,6 +123,7 @@ def serialize_for_delete(uuid: str) -> str: ) except polarion_api.PolarionApiException as error: logger.error("Deleting work items failed. %s", error.args[0]) + raise error def create_missing_work_items( self, converter_session: data_session.ConverterSession @@ -149,6 +150,7 @@ def create_missing_work_items( self.polarion_data_repo.update_work_items(missing_work_items) except polarion_api.PolarionApiException as error: logger.error("Creating work items failed. %s", error.args[0]) + raise error def compare_and_update_work_item( self, converter_data: data_session.ConverterData @@ -205,7 +207,7 @@ def compare_and_update_work_item( *log_args, error.args[0], ) - return + raise error assert new.id is not None delete_links = None @@ -281,6 +283,7 @@ def compare_and_update_work_item( *log_args, error.args[0], ) + raise error def _refactor_attached_images(self, new: data_models.CapellaWorkItem): def set_attachment_id(node: etree._Element) -> None: From 2e172c5b0cf54fe3ff4330ab34377acdbcbea3da Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 15 Aug 2024 12:35:59 +0200 Subject: [PATCH 2/7] feat: more faul tolerance for document renderer --- capella2polarion/__main__.py | 74 ++-------- .../connectors/polarion_worker.py | 44 +++--- .../converters/document_config.py | 6 + .../converters/document_renderer.py | 126 +++++++++++++++++- 4 files changed, 164 insertions(+), 86 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 512867f7..c7c50ec2 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -153,73 +153,27 @@ def render_documents( capella_to_polarion_cli.force_update, ) - polarion_worker.load_polarion_work_item_map() - configs = document_config.read_config_file( document_rendering_config, capella_to_polarion_cli.capella_model ) + + polarion_worker.load_polarion_work_item_map() + documents = polarion_worker.load_polarion_documents( + configs.iterate_documents() + ) + renderer = document_renderer.DocumentRenderer( polarion_worker.polarion_data_repo, capella_to_polarion_cli.capella_model, ) - for config in configs.full_authority: - rendering_layouts = document_config.generate_work_item_layouts( - config.work_item_layouts - ) - for instance in config.instances: - if old_doc := polarion_worker.get_and_customize_document( - instance.polarion_space, - instance.polarion_name, - instance.polarion_title, - rendering_layouts if overwrite_layouts else None, - config.heading_numbering if overwrite_numbering else None, - ): - new_doc, work_items = renderer.render_document( - config.template_directory, - config.template, - document=old_doc, - **instance.params, - ) - polarion_worker.update_document(new_doc) - polarion_worker.update_work_items(work_items) - else: - new_doc, _ = renderer.render_document( - config.template_directory, - config.template, - instance.polarion_space, - instance.polarion_name, - instance.polarion_title, - config.heading_numbering, - rendering_layouts, - **instance.params, - ) - polarion_worker.post_document(new_doc) - - for config in configs.mixed_authority: - rendering_layouts = document_config.generate_work_item_layouts( - config.work_item_layouts - ) - for instance in config.instances: - old_doc = polarion_worker.get_and_customize_document( - instance.polarion_space, - instance.polarion_name, - instance.polarion_title, - rendering_layouts if overwrite_layouts else None, - config.heading_numbering if overwrite_numbering else None, - ) - assert old_doc is not None, ( - "Did not find document " - f"{instance.polarion_space}/{instance.polarion_name}" - ) - new_doc, work_items = renderer.update_mixed_authority_document( - old_doc, - config.template_directory, - config.sections, - instance.params, - instance.section_params, - ) - polarion_worker.update_document(new_doc) - polarion_worker.update_work_items(work_items) + + new_documents, updated_documents, work_items = renderer.render_documents( + configs, documents, overwrite_layouts, overwrite_numbering + ) + + polarion_worker.post_documents(new_documents) + polarion_worker.update_documents(updated_documents) + polarion_worker.update_work_items(work_items) if __name__ == "__main__": diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 4e4c172d..986e54e3 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -425,13 +425,15 @@ def compare_and_update_work_items( if uuid in self.polarion_data_repo and data.work_item is not None: self.compare_and_update_work_item(data) - def post_document(self, document: polarion_api.Document): - """Create a new document.""" - self.client.project_client.documents.create(document) + def post_documents(self, documents: list[polarion_api.Document]): + """Create new documents.""" + if documents: + self.client.project_client.documents.create(documents) - def update_document(self, document: polarion_api.Document): - """Update an existing document.""" - self.client.project_client.documents.update(document) + def update_documents(self, documents: list[polarion_api.Document]): + """Update existing documents.""" + if documents: + self.client.project_client.documents.update(documents) def get_document( self, space: str, name: str @@ -446,24 +448,16 @@ def get_document( return None raise e - def get_and_customize_document( - self, - space: str, - name: str, - new_title: str | None, - rendering_layouts: list[polarion_api.RenderingLayout] | None, - heading_numbering: bool | None, - ) -> polarion_api.Document | None: - """Get a document from polarion and return None if not found.""" - if document := self.get_document(space, name): - document.title = new_title - if rendering_layouts is not None: - document.rendering_layouts = rendering_layouts - if heading_numbering is not None: - document.outline_numbering = heading_numbering - - return document - def update_work_items(self, work_items: list[polarion_api.WorkItem]): """Update the given workitems without any additional checks.""" - self.client.project_client.work_items.update(work_items) + if work_items: + self.client.project_client.work_items.update(work_items) + + def load_polarion_documents( + self, document_paths: t.Iterable[tuple[str, str]] + ) -> dict[tuple[str, str], polarion_api.Document | None]: + """Load the given document references from Polarion.""" + return { + (space, name): self.get_document(space, name) + for space, name in document_paths + } diff --git a/capella2polarion/converters/document_config.py b/capella2polarion/converters/document_config.py index 6311fbee..30ab8df6 100644 --- a/capella2polarion/converters/document_config.py +++ b/capella2polarion/converters/document_config.py @@ -77,6 +77,12 @@ class DocumentConfigs(pydantic.BaseModel): pydantic.Field(default_factory=list) ) + def iterate_documents(self) -> t.Iterator[tuple[str, str]]: + """Yield all document paths of the config as tuples.""" + for conf in self.full_authority + self.mixed_authority: + for inst in conf.instances: + yield inst.polarion_space, inst.polarion_name + def read_config_file( config: t.TextIO, model: capellambse.MelodyModel | None = None diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index 776107a7..06a7b5d4 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -16,7 +16,7 @@ from capella2polarion.connectors import polarion_repo -from . import polarion_html_helper +from . import document_config, polarion_html_helper logger = logging.getLogger(__name__) @@ -286,6 +286,130 @@ def update_mixed_authority_document( return document, session.headings + def render_documents( + self, + configs: document_config.DocumentConfigs, + existing_documents: dict[ + tuple[str, str], polarion_api.Document | None + ], + overwrite_layouts: bool, + overwrite_numbering: bool, + ) -> tuple[ + list[polarion_api.Document], + list[polarion_api.Document], + list[polarion_api.WorkItem], + ]: + """Render all documents defined in the given config. + + Returns a list new documents followed by updated documents and + work items, which need to be updated + """ + + def _get_and_customize_doc( + space: str, + name: str, + title: str | None, + rendering_layouts: list[polarion_api.RenderingLayout], + heading_numbering: bool, + ) -> polarion_api.Document | None: + if old_doc := existing_documents.get((space, name)): + if title: + old_doc.title = title + if overwrite_layouts: + old_doc.rendering_layouts = rendering_layouts + if overwrite_numbering: + old_doc.outline_numbering = heading_numbering + + return old_doc + + new_docs = [] + updated_docs = [] + work_items = [] + for config in configs.full_authority: + rendering_layouts = document_config.generate_work_item_layouts( + config.work_item_layouts + ) + for instance in config.instances: + if old_doc := _get_and_customize_doc( + instance.polarion_space, + instance.polarion_name, + instance.polarion_title, + rendering_layouts, + config.heading_numbering, + ): + try: + new_doc, wis = self.render_document( + config.template_directory, + config.template, + document=old_doc, + **instance.params, + ) + except Exception as e: + logger.error( + "Rendering for document %s/%s failed with the following errors %s", + instance.polarion_space, + instance.polarion_name, + "\n".join(str(e) for e in e.args), + ) + continue + + updated_docs.append(new_doc) + work_items += wis + else: + try: + new_doc, _ = self.render_document( + config.template_directory, + config.template, + instance.polarion_space, + instance.polarion_name, + instance.polarion_title, + config.heading_numbering, + rendering_layouts, + **instance.params, + ) + except Exception as e: + logger.error( + "Rendering for document %s/%s failed with the following errors %s", + instance.polarion_space, + instance.polarion_name, + "\n".join(str(e) for e in e.args), + ) + continue + + new_docs.append(new_doc) + + for config in configs.mixed_authority: + rendering_layouts = document_config.generate_work_item_layouts( + config.work_item_layouts + ) + for instance in config.instances: + old_doc = _get_and_customize_doc( + instance.polarion_space, + instance.polarion_name, + instance.polarion_title, + rendering_layouts, + config.heading_numbering, + ) + if old_doc is None: + logger.error( + "For document %s/%s no document was found, but it's mandatory to have one in mixed authority mode", + instance.polarion_space, + instance.polarion_name, + ) + continue + + new_doc, work_items = self.update_mixed_authority_document( + old_doc, + config.template_directory, + config.sections, + instance.params, + instance.section_params, + ) + updated_docs.append(new_doc) + work_items += work_items + + return new_docs, updated_docs, work_items + def _extract_section_areas(self, html_elements: list[etree._Element]): section_areas = {} current_area_id = None From 888a787c7d6fde43722445f4baf61818aacb3b31 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 15 Aug 2024 13:21:15 +0200 Subject: [PATCH 3/7] chore: Use exc_info to log the error --- .../converters/document_renderer.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index 06a7b5d4..73250648 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -346,10 +346,10 @@ def _get_and_customize_doc( ) except Exception as e: logger.error( - "Rendering for document %s/%s failed with the following errors %s", + "Rendering for document %s/%s failed with the following error", instance.polarion_space, instance.polarion_name, - "\n".join(str(e) for e in e.args), + exc_info=e, ) continue @@ -369,10 +369,10 @@ def _get_and_customize_doc( ) except Exception as e: logger.error( - "Rendering for document %s/%s failed with the following errors %s", + "Rendering for document %s/%s failed with the following error", instance.polarion_space, instance.polarion_name, - "\n".join(str(e) for e in e.args), + exc_info=e, ) continue @@ -397,14 +397,23 @@ def _get_and_customize_doc( instance.polarion_name, ) continue + try: + new_doc, work_items = self.update_mixed_authority_document( + old_doc, + config.template_directory, + config.sections, + instance.params, + instance.section_params, + ) + except Exception as e: + logger.error( + "Rendering for document %s/%s failed with the following error", + instance.polarion_space, + instance.polarion_name, + exc_info=e, + ) + continue - new_doc, work_items = self.update_mixed_authority_document( - old_doc, - config.template_directory, - config.sections, - instance.params, - instance.section_params, - ) updated_docs.append(new_doc) work_items += work_items From e17a259d7b210bc41c2e309cef5fd1133c24b044 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Thu, 15 Aug 2024 19:50:09 +0200 Subject: [PATCH 4/7] chore: Refactor document rendering and add additional tests including a CLI test --- capella2polarion/__main__.py | 4 +- .../converters/document_renderer.py | 180 +++++++++++------- .../converters/polarion_html_helper.py | 3 + tests/conftest.py | 2 + tests/data/documents/combined_config.yaml | 36 ++-- tests/test_cli.py | 76 +++++++- tests/test_documents.py | 110 +++++++++-- 7 files changed, 303 insertions(+), 108 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index c7c50ec2..8f924768 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -165,10 +165,12 @@ def render_documents( renderer = document_renderer.DocumentRenderer( polarion_worker.polarion_data_repo, capella_to_polarion_cli.capella_model, + overwrite_numbering, + overwrite_layouts, ) new_documents, updated_documents, work_items = renderer.render_documents( - configs, documents, overwrite_layouts, overwrite_numbering + configs, documents ) polarion_worker.post_documents(new_documents) diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index 73250648..b3ba7180 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -44,10 +44,14 @@ def __init__( self, polarion_repository: polarion_repo.PolarionDataRepository, model: capellambse.MelodyModel, + overwrite_heading_numbering: bool = False, + overwrite_layouts: bool = False, ): self.polarion_repository = polarion_repository self.model = model self.jinja_envs: dict[str, jinja2.Environment] = {} + self.overwrite_heading_numbering = overwrite_heading_numbering + self.overwrite_layouts = overwrite_layouts def setup_env(self, env: jinja2.Environment): """Add globals and filters to the environment.""" @@ -238,7 +242,7 @@ def update_mixed_authority_document( section_areas = self._extract_section_areas(html_elements) session = RenderingSession( - rendering_layouts=document.rendering_layouts + rendering_layouts=document.rendering_layouts or [] ) env = self._get_jinja_env(template_folder) @@ -286,14 +290,33 @@ def update_mixed_authority_document( return document, session.headings + def _get_and_customize_doc( + self, + space: str, + name: str, + title: str | None, + rendering_layouts: list[polarion_api.RenderingLayout], + heading_numbering: bool, + existing_documents: dict[ + tuple[str, str], polarion_api.Document | None + ], + ) -> polarion_api.Document | None: + if old_doc := existing_documents.get((space, name)): + if title: + old_doc.title = title + if self.overwrite_layouts: + old_doc.rendering_layouts = rendering_layouts + if self.overwrite_heading_numbering: + old_doc.outline_numbering = heading_numbering + + return old_doc + def render_documents( self, configs: document_config.DocumentConfigs, existing_documents: dict[ tuple[str, str], polarion_api.Document | None ], - overwrite_layouts: bool, - overwrite_numbering: bool, ) -> tuple[ list[polarion_api.Document], list[polarion_api.Document], @@ -305,37 +328,99 @@ def render_documents( work items, which need to be updated """ - def _get_and_customize_doc( - space: str, - name: str, - title: str | None, - rendering_layouts: list[polarion_api.RenderingLayout], - heading_numbering: bool, - ) -> polarion_api.Document | None: - if old_doc := existing_documents.get((space, name)): - if title: - old_doc.title = title - if overwrite_layouts: - old_doc.rendering_layouts = rendering_layouts - if overwrite_numbering: - old_doc.outline_numbering = heading_numbering - - return old_doc - - new_docs = [] - updated_docs = [] - work_items = [] - for config in configs.full_authority: + new_docs: list[polarion_api.Document] = [] + updated_docs: list[polarion_api.Document] = [] + work_items: list[polarion_api.WorkItem] = [] + self._render_full_authority_documents( + configs.full_authority, + existing_documents, + new_docs, + updated_docs, + work_items, + ) + + self._render_mixed_authority_documents( + configs.mixed_authority, + existing_documents, + updated_docs, + work_items, + ) + + return new_docs, updated_docs, work_items + + def _render_mixed_authority_documents( + self, + mixed_authority_configs: list[ + document_config.FullAuthorityDocumentRenderingConfig + ], + existing_documents: dict[ + tuple[str, str], polarion_api.Document | None + ], + updated_docs: list[polarion_api.Document], + work_items: list[polarion_api.WorkItem], + ): + for config in mixed_authority_configs: rendering_layouts = document_config.generate_work_item_layouts( config.work_item_layouts ) for instance in config.instances: - if old_doc := _get_and_customize_doc( + old_doc = self._get_and_customize_doc( instance.polarion_space, instance.polarion_name, instance.polarion_title, rendering_layouts, config.heading_numbering, + existing_documents, + ) + if old_doc is None: + logger.error( + "For document %s/%s no document was found, but it's mandatory to have one in mixed authority mode", + instance.polarion_space, + instance.polarion_name, + ) + continue + try: + new_doc, wis = self.update_mixed_authority_document( + old_doc, + config.template_directory, + config.sections, + instance.params, + instance.section_params, + ) + except Exception as e: + logger.error( + "Rendering for document %s/%s failed with the following error", + instance.polarion_space, + instance.polarion_name, + exc_info=e, + ) + continue + + updated_docs.append(new_doc) + work_items.extend(wis) + + def _render_full_authority_documents( + self, + full_authority_configs, + existing_documents: dict[ + tuple[str, str], polarion_api.Document | None + ], + new_docs: list[polarion_api.Document], + updated_docs: list[polarion_api.Document], + work_items: list[polarion_api.WorkItem], + ): + for config in full_authority_configs: + rendering_layouts = document_config.generate_work_item_layouts( + config.work_item_layouts + ) + for instance in config.instances: + if old_doc := self._get_and_customize_doc( + instance.polarion_space, + instance.polarion_name, + instance.polarion_title, + rendering_layouts, + config.heading_numbering, + existing_documents, ): try: new_doc, wis = self.render_document( @@ -354,7 +439,7 @@ def _get_and_customize_doc( continue updated_docs.append(new_doc) - work_items += wis + work_items.extend(wis) else: try: new_doc, _ = self.render_document( @@ -376,48 +461,7 @@ def _get_and_customize_doc( ) continue - new_docs.append(new_doc) - - for config in configs.mixed_authority: - rendering_layouts = document_config.generate_work_item_layouts( - config.work_item_layouts - ) - for instance in config.instances: - old_doc = _get_and_customize_doc( - instance.polarion_space, - instance.polarion_name, - instance.polarion_title, - rendering_layouts, - config.heading_numbering, - ) - if old_doc is None: - logger.error( - "For document %s/%s no document was found, but it's mandatory to have one in mixed authority mode", - instance.polarion_space, - instance.polarion_name, - ) - continue - try: - new_doc, work_items = self.update_mixed_authority_document( - old_doc, - config.template_directory, - config.sections, - instance.params, - instance.section_params, - ) - except Exception as e: - logger.error( - "Rendering for document %s/%s failed with the following error", - instance.polarion_space, - instance.polarion_name, - exc_info=e, - ) - continue - - updated_docs.append(new_doc) - work_items += work_items - - return new_docs, updated_docs, work_items + new_docs.append(new_doc) def _extract_section_areas(self, html_elements: list[etree._Element]): section_areas = {} diff --git a/capella2polarion/converters/polarion_html_helper.py b/capella2polarion/converters/polarion_html_helper.py index 5b29fcdc..9d7fa573 100644 --- a/capella2polarion/converters/polarion_html_helper.py +++ b/capella2polarion/converters/polarion_html_helper.py @@ -143,6 +143,9 @@ def extract_headings(html_content: str | list[etree._Element]) -> list[str]: html_fragments = _ensure_fragments(html_content) for element in html_fragments: + if isinstance(element, html.HtmlComment): + continue + if h_regex.fullmatch(element.tag): if matches := wi_regex.match(element.get("id")): heading_ids.append(matches.group(1)) diff --git a/tests/conftest.py b/tests/conftest.py index 07e4b6a6..a1310ecf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,8 @@ TEST_DIAGRAM_CACHE = TEST_DATA_ROOT / "diagram_cache" TEST_MODEL_ELEMENTS = TEST_DATA_ROOT / "model_elements" TEST_MODEL_ELEMENTS_CONFIG = TEST_MODEL_ELEMENTS / "config.yaml" +TEST_DOCUMENT_ROOT = TEST_DATA_ROOT / "documents" +TEST_COMBINED_DOCUMENT_CONFIG = TEST_DOCUMENT_ROOT / "combined_config.yaml" TEST_MODEL = { "path": str(TEST_DATA_ROOT / "model" / "Melody Model Test.aird"), "diagram_cache": str(TEST_DIAGRAM_CACHE), diff --git a/tests/data/documents/combined_config.yaml b/tests/data/documents/combined_config.yaml index 923905f1..6a1034b3 100644 --- a/tests/data/documents/combined_config.yaml +++ b/tests/data/documents/combined_config.yaml @@ -2,25 +2,20 @@ # SPDX-License-Identifier: Apache-2.0 mixed_authority: - - template_directory: jupyter-notebooks/document_templates + - template_directory: tests/data/documents/sections sections: - section1: test-icd.html.j2 - section2: test-icd.html.j2 + section1: section1.html.j2 + section2: section2.html.j2 instances: - polarion_space: _default polarion_name: id123 polarion_title: Interface23 params: - interface: 3d21ab4b-7bf6-428b-ba4c-a27bca4e86db + interface: 4b5bea95-9bc2-477c-a8b2-c4e54b5066fb - polarion_space: _default polarion_name: id1234 params: interface: 3d21ab4b-7bf6-428b-ba4c-a27bca4e86db - - template_directory: jupyter-notebooks/document_templates - sections: - section1: test-icd.html.j2 - section2: test-icd.html.j2 - heading_numbering: True work_item_layouts: componentExchange: fields_at_start: @@ -31,9 +26,14 @@ mixed_authority: show_title: False fields_at_end: - tree_view_diagram + - template_directory: jupyter-notebooks/document_templates + sections: + section1: test-icd.html.j2 + section2: test-icd.html.j2 + heading_numbering: True instances: - polarion_space: _default - polarion_name: id1234 + polarion_name: id1235 section_params: section1: param_1: Test @@ -42,17 +42,14 @@ full_authority: template: test-icd.html.j2 instances: - polarion_space: _default - polarion_name: id123 + polarion_name: id1236 polarion_title: Interface23 params: - interface: 3d21ab4b-7bf6-428b-ba4c-a27bca4e86db + interface: 4b5bea95-9bc2-477c-a8b2-c4e54b5066fb - polarion_space: _default - polarion_name: id1234 + polarion_name: id1237 params: - interface: 3d21ab4b-7bf6-428b-ba4c-a27bca4e86db - - template_directory: jupyter-notebooks/document_templates - template: test-no-args.html.j2 - heading_numbering: True + interface: 2681f26a-e492-4e5d-8b33-92fb00a48622 work_item_layouts: componentExchange: fields_at_start: @@ -63,6 +60,9 @@ full_authority: show_title: False fields_at_end: - tree_view_diagram + - template_directory: jupyter-notebooks/document_templates + template: test-no-args.html.j2 + heading_numbering: True instances: - polarion_space: _default - polarion_name: id1234 + polarion_name: id1238 diff --git a/tests/test_cli.py b/tests/test_cli.py index f71dd60c..2620374d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,6 +15,7 @@ # pylint: disable-next=relative-beyond-top-level, useless-suppression from .conftest import ( # type: ignore[import] + TEST_COMBINED_DOCUMENT_CONFIG, TEST_MODEL, TEST_MODEL_ELEMENTS_CONFIG, ) @@ -66,8 +67,81 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): result = testing.CliRunner().invoke(main.cli, command, terminal_width=60) assert result.exit_code == 0 - assert mock_get_polarion_wi_map.call_count == 1 assert mock_delete_work_items.call_count == 1 assert mock_patch_work_items.call_count == 1 assert mock_post_work_items.call_count == 1 + + +def test_render_documents(monkeypatch: pytest.MonkeyPatch): + mock_api = mock.MagicMock(spec=polarion_api.OpenAPIPolarionProjectClient) + monkeypatch.setattr(polarion_api, "OpenAPIPolarionProjectClient", mock_api) + mock_get_polarion_wi_map = mock.MagicMock() + monkeypatch.setattr( + CapellaPolarionWorker, + "load_polarion_work_item_map", + mock_get_polarion_wi_map, + ) + mock_get_document = mock.MagicMock() + mock_get_document.side_effect = lambda folder, name: ( + polarion_api.Document( + module_folder=folder, + module_name=name, + home_page_content=polarion_api.TextContent( + "text/html", + '

dict[tuple[str, str], polarion_api.Document]: + return { + ("_default", "id123"): polarion_api.Document( + module_folder="_default", + module_name="id123", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), + rendering_layouts=[ + polarion_api.RenderingLayout( + "Class", "paragraph", type="class" + ) + ], + ), + ("_default", "id1237"): polarion_api.Document( + module_folder="_default", + module_name="id1237", + home_page_content=polarion_api.TextContent( + type="text/html", + value=MIXED_AUTHORITY_DOCUMENT.read_text("utf-8"), + ), + ), + } def test_create_new_document( @@ -41,8 +70,8 @@ def test_create_new_document( ) new_doc, wis = renderer.render_document( - "jupyter-notebooks/document_templates", - "test-classes.html.j2", + JUPYTER_TEMPLATE_FOLDER, + CLASSES_TEMPLATE, "_default", "TEST-DOC", cls="c710f1c2-ede6-444e-9e2b-0ff30d7fd040", @@ -112,8 +141,8 @@ def test_update_document( ) new_doc, wis = renderer.render_document( - "jupyter-notebooks/document_templates", - "test-classes.html.j2", + JUPYTER_TEMPLATE_FOLDER, + CLASSES_TEMPLATE, document=old_doc, cls="c710f1c2-ede6-444e-9e2b-0ff30d7fd040", ) @@ -192,19 +221,61 @@ def test_mixed_authority_document( assert wis[0].title == "Keep Heading" +def test_render_all_documents_partially_successfully( + empty_polarion_worker: polarion_worker.CapellaPolarionWorker, + model: capellambse.MelodyModel, + caplog: pytest.LogCaptureFixture, +): + with open(TEST_COMBINED_DOCUMENT_CONFIG, "r", encoding="utf-8") as f: + conf = document_config.read_config_file(f) + + renderer = document_renderer.DocumentRenderer( + empty_polarion_worker.polarion_data_repo, model + ) + + new_docs, updated_docs, work_items = renderer.render_documents( + conf, existing_documents() + ) + + assert len(caplog.records) == 3 + assert len(new_docs) == 1 + assert len(updated_docs) == 2 + assert len(work_items) == 3 + assert len(updated_docs[0].rendering_layouts) == 0 + assert len(updated_docs[1].rendering_layouts) == 1 + assert updated_docs[0].outline_numbering is None + assert updated_docs[1].outline_numbering is None + + +def test_render_all_documents_overwrite_headings_layouts( + empty_polarion_worker: polarion_worker.CapellaPolarionWorker, + model: capellambse.MelodyModel, +): + with open(TEST_COMBINED_DOCUMENT_CONFIG, "r", encoding="utf-8") as f: + conf = document_config.read_config_file(f) + + renderer = document_renderer.DocumentRenderer( + empty_polarion_worker.polarion_data_repo, model, True, True + ) + + _, updated_docs, _ = renderer.render_documents(conf, existing_documents()) + + assert len(updated_docs[0].rendering_layouts) == 2 + assert len(updated_docs[1].rendering_layouts) == 2 + assert updated_docs[0].outline_numbering is False + assert updated_docs[1].outline_numbering is False + + def test_full_authority_document_config(): with open( - "tests/data/documents/full_authority_config.yaml", + FULL_AUTHORITY_CONFIG, "r", encoding="utf-8", ) as f: conf = document_config.read_config_file(f) assert len(conf.full_authority) == 2 - assert ( - conf.full_authority[0].template_directory - == "jupyter-notebooks/document_templates" - ) + assert conf.full_authority[0].template_directory == JUPYTER_TEMPLATE_FOLDER assert conf.full_authority[0].template == "test-icd.html.j2" assert conf.full_authority[0].heading_numbering is False assert len(conf.full_authority[0].instances) == 2 @@ -223,8 +294,7 @@ def test_mixed_authority_document_config(): assert len(conf.full_authority) == 0 assert len(conf.mixed_authority) == 2 assert ( - conf.mixed_authority[0].template_directory - == "jupyter-notebooks/document_templates" + conf.mixed_authority[0].template_directory == JUPYTER_TEMPLATE_FOLDER ) assert conf.mixed_authority[0].sections == { "section1": "test-icd.html.j2", @@ -244,7 +314,7 @@ def test_mixed_authority_document_config(): def test_combined_config(): - with open(COMBINED_CONFIG, "r", encoding="utf-8") as f: + with open(TEST_COMBINED_DOCUMENT_CONFIG, "r", encoding="utf-8") as f: conf = document_config.read_config_file(f) assert len(conf.full_authority) == 2 From c2e2e5077571b61e0199c60e5f89ce9bb320fd44 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 16 Aug 2024 08:14:32 +0200 Subject: [PATCH 5/7] chore: add some comments in a new test case --- tests/test_documents.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_documents.py b/tests/test_documents.py index 9016c703..9c56332e 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -11,9 +11,7 @@ from tests.conftest import TEST_COMBINED_DOCUMENT_CONFIG, TEST_DOCUMENT_ROOT CLASSES_TEMPLATE = "test-classes.html.j2" - JUPYTER_TEMPLATE_FOLDER = "jupyter-notebooks/document_templates" - DOCUMENT_SECTIONS = TEST_DOCUMENT_ROOT / "sections" MIXED_CONFIG = TEST_DOCUMENT_ROOT / "mixed_config.yaml" FULL_AUTHORITY_CONFIG = TEST_DOCUMENT_ROOT / "full_authority_config.yaml" @@ -237,9 +235,15 @@ def test_render_all_documents_partially_successfully( conf, existing_documents() ) + # There are 6 documents in the config, we expect 3 rendering to fail assert len(caplog.records) == 3 + # For one valid config we did not pass a document, so we expect a new one assert len(new_docs) == 1 + # And two updated documents assert len(updated_docs) == 2 + # In both existing documents we had 2 headings. In full authority mode + # both should be updated and in mixed authority mode only one of them as + # the other is outside the rendering area assert len(work_items) == 3 assert len(updated_docs[0].rendering_layouts) == 0 assert len(updated_docs[1].rendering_layouts) == 1 From 96d47c4d55b5ff08f135817557b8415470aae04a Mon Sep 17 00:00:00 2001 From: micha91 Date: Fri, 16 Aug 2024 15:08:17 +0200 Subject: [PATCH 6/7] chore: Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ernst Würger --- capella2polarion/converters/document_renderer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/capella2polarion/converters/document_renderer.py b/capella2polarion/converters/document_renderer.py index b3ba7180..04d3bc3c 100644 --- a/capella2polarion/converters/document_renderer.py +++ b/capella2polarion/converters/document_renderer.py @@ -374,7 +374,8 @@ def _render_mixed_authority_documents( ) if old_doc is None: logger.error( - "For document %s/%s no document was found, but it's mandatory to have one in mixed authority mode", + "For document %s/%s no document was found, but it's " + "mandatory to have one in mixed authority mode", instance.polarion_space, instance.polarion_name, ) @@ -389,7 +390,8 @@ def _render_mixed_authority_documents( ) except Exception as e: logger.error( - "Rendering for document %s/%s failed with the following error", + "Rendering for document %s/%s failed with the " + "following error", instance.polarion_space, instance.polarion_name, exc_info=e, From 4282892206ffdf2a066475896339f7d3549dd181 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Fri, 16 Aug 2024 15:20:38 +0200 Subject: [PATCH 7/7] chore: remove unnecessary checks --- capella2polarion/connectors/polarion_worker.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/capella2polarion/connectors/polarion_worker.py b/capella2polarion/connectors/polarion_worker.py index 986e54e3..b60ae11d 100644 --- a/capella2polarion/connectors/polarion_worker.py +++ b/capella2polarion/connectors/polarion_worker.py @@ -427,13 +427,11 @@ def compare_and_update_work_items( def post_documents(self, documents: list[polarion_api.Document]): """Create new documents.""" - if documents: - self.client.project_client.documents.create(documents) + self.client.project_client.documents.create(documents) def update_documents(self, documents: list[polarion_api.Document]): """Update existing documents.""" - if documents: - self.client.project_client.documents.update(documents) + self.client.project_client.documents.update(documents) def get_document( self, space: str, name: str @@ -450,8 +448,7 @@ def get_document( def update_work_items(self, work_items: list[polarion_api.WorkItem]): """Update the given workitems without any additional checks.""" - if work_items: - self.client.project_client.work_items.update(work_items) + self.client.project_client.work_items.update(work_items) def load_polarion_documents( self, document_paths: t.Iterable[tuple[str, str]]