From b80e7ba28d02cc3285fc01327ef0f8ca0b514a73 Mon Sep 17 00:00:00 2001 From: Adrian Braemer Date: Thu, 16 Jan 2025 14:31:47 +0100 Subject: [PATCH 1/2] refactor: convert all field names to snake_case using alias_generator from pydantic --- src/opossum_lib/opossum/file_generation.py | 2 +- src/opossum_lib/opossum/merger.py | 26 +++--- src/opossum_lib/opossum/opossum_file.py | 89 ++++++++++--------- src/opossum_lib/opossum/output_model.py | 2 +- .../scancode/convert_scancode_to_opossum.py | 14 +-- src/opossum_lib/scancode/resource_tree.py | 2 +- src/opossum_lib/spdx/convert_to_opossum.py | 10 +-- tests/test_opossum/test_file_generation.py | 22 ++--- tests/test_opossum/test_merge.py | 38 ++++---- ...t_extract_opossum_information_from_spdx.py | 24 ++--- 10 files changed, 117 insertions(+), 112 deletions(-) diff --git a/src/opossum_lib/opossum/file_generation.py b/src/opossum_lib/opossum/file_generation.py index 6db6daf..cf1ae9f 100644 --- a/src/opossum_lib/opossum/file_generation.py +++ b/src/opossum_lib/opossum/file_generation.py @@ -45,6 +45,6 @@ def write_input_json( zip_file.writestr( INPUT_JSON_NAME, TypeAdapter(OpossumInformation).dump_json( - opossum_file_content.input_file, indent=4, exclude_none=True + opossum_file_content.input_file, indent=4, exclude_none=True, by_alias=True ), ) diff --git a/src/opossum_lib/opossum/merger.py b/src/opossum_lib/opossum/merger.py index fc4f422..c87dcb6 100644 --- a/src/opossum_lib/opossum/merger.py +++ b/src/opossum_lib/opossum/merger.py @@ -28,27 +28,27 @@ def merge_opossum_information( for opossum_information in expanded_opossum_information ] ).convert_to_file_resource(), - externalAttributions=_merge_dicts_without_duplicates( + external_attributions=_merge_dicts_without_duplicates( [ - opossum_information.externalAttributions + opossum_information.external_attributions for opossum_information in expanded_opossum_information ] ), - resourcesToAttributions=_merge_resources_to_attributions( + resources_to_attributions=_merge_resources_to_attributions( [ - opossum_information.resourcesToAttributions + opossum_information.resources_to_attributions for opossum_information in expanded_opossum_information ] ), attributionBreakpoints=_merge_attribution_breakpoints( [ - opossum_information.attributionBreakpoints + opossum_information.attribution_breakpoints for opossum_information in expanded_opossum_information ] ), externalAttributionSources=_merge_dicts_without_duplicates( [ - opossum_information.externalAttributionSources + opossum_information.external_attribution_sources for opossum_information in expanded_opossum_information ] ), @@ -61,12 +61,12 @@ def expand_opossum_package_identifier( """IDs for the attributions should be unique per OpossumInformation. To prevent possible duplicates we add the projectId of the OpossumInformation to the IDs as a prefix.""" - prefix = opossum_information.metadata.projectId + prefix = opossum_information.metadata.project_id extended_resources_to_attributions = dict() for ( resource_path, identifiers, - ) in opossum_information.resourcesToAttributions.items(): + ) in opossum_information.resources_to_attributions.items(): extended_resources_to_attributions[resource_path] = [ prefix + "-" + identifier for identifier in identifiers ] @@ -74,16 +74,16 @@ def expand_opossum_package_identifier( for ( identifier, external_attribution, - ) in opossum_information.externalAttributions.items(): + ) in opossum_information.external_attributions.items(): extended_external_attributions[prefix + "-" + identifier] = external_attribution return OpossumInformation( metadata=opossum_information.metadata, resources=opossum_information.resources, - externalAttributions=extended_external_attributions, - resourcesToAttributions=extended_resources_to_attributions, - attributionBreakpoints=opossum_information.attributionBreakpoints, - externalAttributionSources=opossum_information.externalAttributionSources, + external_attributions=extended_external_attributions, + resources_to_attributions=extended_resources_to_attributions, + attributionBreakpoints=opossum_information.attribution_breakpoints, + externalAttributionSources=opossum_information.external_attribution_sources, ) diff --git a/src/opossum_lib/opossum/opossum_file.py b/src/opossum_lib/opossum/opossum_file.py index c557e2d..e7d1b06 100644 --- a/src/opossum_lib/opossum/opossum_file.py +++ b/src/opossum_lib/opossum/opossum_file.py @@ -9,28 +9,33 @@ from typing import Literal, cast from pydantic import BaseModel, ConfigDict, model_serializer +from pydantic.alias_generators import to_camel type OpossumPackageIdentifier = str type ResourcePath = str type ResourceInFile = dict[str, ResourceInFile] | int -class OpossumInformation(BaseModel): +class CamelBaseModel(BaseModel): + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + +class OpossumInformation(CamelBaseModel): model_config = ConfigDict(frozen=True) metadata: Metadata resources: ResourceInFile - externalAttributions: dict[OpossumPackageIdentifier, OpossumPackage] - resourcesToAttributions: dict[ResourcePath, list[OpossumPackageIdentifier]] - attributionBreakpoints: list[str] = field(default_factory=list) - externalAttributionSources: dict[str, ExternalAttributionSource] = field( + external_attributions: dict[OpossumPackageIdentifier, OpossumPackage] + resources_to_attributions: dict[ResourcePath, list[OpossumPackageIdentifier]] + attribution_breakpoints: list[str] = field(default_factory=list) + external_attribution_sources: dict[str, ExternalAttributionSource] = field( default_factory=dict ) - frequentLicenses: list[FrequentLicense] | None = None - filesWithChildren: list[str] | None = None - baseUrlsForSources: BaseUrlsForSources | None = None + frequent_licenses: list[FrequentLicense] | None = None + files_with_children: list[str] | None = None + base_urls_for_sources: BaseUrlsForSources | None = None -class BaseUrlsForSources(BaseModel): +class BaseUrlsForSources(CamelBaseModel): @model_serializer def serialize(self) -> dict: # hack to override not serializing keys with corresponding value none: @@ -40,51 +45,51 @@ def serialize(self) -> dict: model_config = ConfigDict(extra="allow", frozen=True) -class FrequentLicense(BaseModel): - fullName: str - shortName: str - defaultText: str +class FrequentLicense(CamelBaseModel): + full_name: str + short_name: str + default_text: str -class SourceInfo(BaseModel): +class SourceInfo(CamelBaseModel): model_config = ConfigDict(frozen=True) name: str - documentConfidence: int | float | None = 0 - additionalName: str | None = None + document_confidence: int | float | None = 0 + additional_name: str | None = None -class OpossumPackage(BaseModel): +class OpossumPackage(CamelBaseModel): model_config = ConfigDict(frozen=True) source: SourceInfo - attributionConfidence: int | None = None + attribution_confidence: int | None = None comment: str | None = None - packageName: str | None = None - packageVersion: str | None = None - packageNamespace: str | None = None - packageType: str | None = None - packagePURLAppendix: str | None = None + package_name: str | None = None + package_version: str | None = None + package_namespace: str | None = None + package_type: str | None = None + package_p_u_r_l_appendix: str | None = None copyright: str | None = None - licenseName: str | None = None - licenseText: str | None = None + license_name: str | None = None + license_text: str | None = None url: str | None = None - firstParty: bool | None = None - excludeFromNotice: bool | None = None - preSelected: bool | None = None - followUp: Literal["FOLLOW_UP"] | None = None - originId: str | None = None - originIds: list[str] | None = None + first_party: bool | None = None + exclude_from_notice: bool | None = None + pre_selected: bool | None = None + follow_up: Literal["FOLLOW_UP"] | None = None + origin_id: str | None = None + origin_ids: list[str] | None = None criticality: Literal["high"] | Literal["medium"] | None = None - wasPreferred: bool | None = None + was_preferred: bool | None = None -class Metadata(BaseModel): +class Metadata(CamelBaseModel): model_config = ConfigDict(extra="allow", frozen=True) - projectId: str - fileCreationDate: str - projectTitle: str - projectVersion: str | None = None - expectedReleaseDate: str | None = None - buildDate: str | None = None + project_id: str + file_creation_date: str + project_title: str + project_version: str | None = None + expected_release_date: str | None = None + build_date: str | None = None class ResourceType(Enum): @@ -94,7 +99,7 @@ class ResourceType(Enum): OTHER = auto() -class Resource(BaseModel): +class Resource(CamelBaseModel): model_config = ConfigDict(frozen=True) type: ResourceType children: dict[str, Resource] = field(default_factory=dict) @@ -181,11 +186,11 @@ def convert_to_file_resource(self) -> ResourceInFile: return self.to_dict() -class ExternalAttributionSource(BaseModel): +class ExternalAttributionSource(CamelBaseModel): model_config = ConfigDict(frozen=True) name: str priority: int - isRelevantForPreferred: bool | None = None + is_relevant_for_preferred: bool | None = None def _build_resource_tree(resource: ResourceInFile) -> Resource: diff --git a/src/opossum_lib/opossum/output_model.py b/src/opossum_lib/opossum/output_model.py index 71909c5..8addd5c 100644 --- a/src/opossum_lib/opossum/output_model.py +++ b/src/opossum_lib/opossum/output_model.py @@ -14,7 +14,7 @@ class CamelBaseModel(BaseModel): - model_config = ConfigDict(alias_generator=to_camel) + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) class Metadata(CamelBaseModel): diff --git a/src/opossum_lib/scancode/convert_scancode_to_opossum.py b/src/opossum_lib/scancode/convert_scancode_to_opossum.py index e3a94d2..81db539 100644 --- a/src/opossum_lib/scancode/convert_scancode_to_opossum.py +++ b/src/opossum_lib/scancode/convert_scancode_to_opossum.py @@ -34,19 +34,19 @@ def convert_scancode_to_opossum(filename: str) -> OpossumFileContent: scancode_header = extract_scancode_header(scancode_data, filename) metadata = Metadata( - projectId=str(uuid.uuid4()), - fileCreationDate=scancode_header.end_timestamp, - projectTitle="ScanCode file", + project_id=str(uuid.uuid4()), + file_creation_date=scancode_header.end_timestamp, + project_title="ScanCode file", ) return OpossumFileContent( OpossumInformation( metadata=metadata, resources=resources, - externalAttributions=external_attributions, - resourcesToAttributions=resources_to_attributions, - attributionBreakpoints=[], - externalAttributionSources={}, + external_attributions=external_attributions, + resources_to_attributions=resources_to_attributions, + attribution_breakpoints=[], + external_attribution_sources={}, ) ) diff --git a/src/opossum_lib/scancode/resource_tree.py b/src/opossum_lib/scancode/resource_tree.py index 6c337b6..ad1e898 100644 --- a/src/opossum_lib/scancode/resource_tree.py +++ b/src/opossum_lib/scancode/resource_tree.py @@ -90,7 +90,7 @@ def get_attribution_info(file: File) -> list[OpossumPackage]: def get_attribution_key(attribution: OpossumPackage) -> OpossumPackageIdentifier: - return f"{attribution.licenseName}-{hash(attribution)}" + return f"{attribution.license_name}-{hash(attribution)}" def create_attribution_mapping( diff --git a/src/opossum_lib/spdx/convert_to_opossum.py b/src/opossum_lib/spdx/convert_to_opossum.py index e413457..4ef2c1d 100644 --- a/src/opossum_lib/spdx/convert_to_opossum.py +++ b/src/opossum_lib/spdx/convert_to_opossum.py @@ -121,8 +121,8 @@ def convert_tree_to_opossum_information(tree: DiGraph) -> OpossumInformation: opossum_information = OpossumInformation( metadata=metadata, resources=resources.convert_to_file_resource(), - externalAttributions=external_attributions, - resourcesToAttributions=resources_to_attributions, + external_attributions=external_attributions, + resources_to_attributions=resources_to_attributions, attributionBreakpoints=attribution_breakpoints, externalAttributionSources=external_attribution_sources, ) @@ -175,8 +175,8 @@ def create_metadata(tree: DiGraph) -> Metadata: doc_name = tree.nodes["SPDXRef-DOCUMENT"]["element"].name created = tree.nodes["SPDXRef-DOCUMENT"]["element"].created metadata = Metadata( - projectId=str(uuid.uuid4()), - fileCreationDate=created.isoformat(), - projectTitle=doc_name, + project_id=str(uuid.uuid4()), + file_creation_date=created.isoformat(), + project_title=doc_name, ) return metadata diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index cf223e8..3c867ae 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -14,24 +14,24 @@ OPOSSUM_INFILE = OpossumInformation( metadata=Metadata( - projectId="project-id", - fileCreationDate="30-05-2023", - projectTitle="project-title", + project_id="project-id", + file_creation_date="30-05-2023", + project_title="project-title", ), resources={}, - externalAttributions={}, - resourcesToAttributions={}, + external_attributions={}, + resources_to_attributions={}, ) OPOSSUM_OUTFILE = OpossumOutputFile( metadata=opossum_lib.opossum.output_model.Metadata( - projectId="project-id", - fileCreationDate="30-05-2023", - inputFileMd5Checksum="checksum", + project_id="project-id", + file_creation_date="30-05-2023", + input_file_md5_checksum="checksum", ), - manualAttributions={}, - resourcesToAttributions={}, - resolvedExternalAttributions=None, + manual_attributions={}, + resources_to_attributions={}, + resolved_external_attributions=None, ) diff --git a/tests/test_opossum/test_merge.py b/tests/test_opossum/test_merge.py index 981080d..529f82b 100644 --- a/tests/test_opossum/test_merge.py +++ b/tests/test_opossum/test_merge.py @@ -28,24 +28,24 @@ def test_merge_opossum_information() -> None: opossum_package = OpossumPackage(source=SourceInfo(name="source")) opossum_information = OpossumInformation( metadata=Metadata( - projectId="project-id", - fileCreationDate="30-05-2023", - projectTitle="test data", + project_id="project-id", + file_creation_date="30-05-2023", + project_title="test data", ), resources={"A": {"B": {}}}, - externalAttributions={"SPDXRef-Package": opossum_package}, - resourcesToAttributions={"/A/B/": ["SPDXRef-Package"]}, + external_attributions={"SPDXRef-Package": opossum_package}, + resources_to_attributions={"/A/B/": ["SPDXRef-Package"]}, ) opossum_information_2 = OpossumInformation( metadata=Metadata( - projectId="test-data-id", - fileCreationDate="29-05-2023", - projectTitle="second test data", + project_id="test-data-id", + file_creation_date="29-05-2023", + project_title="second test data", ), resources={"A": {"D": {"C": 1}}}, - externalAttributions={"SPDXRef-File": opossum_package}, - resourcesToAttributions={"/A/D/C": ["SPDXRef-File"]}, + external_attributions={"SPDXRef-File": opossum_package}, + resources_to_attributions={"/A/D/C": ["SPDXRef-File"]}, ) merged_information = merge_opossum_information( @@ -59,11 +59,11 @@ def test_merge_opossum_information() -> None: "D": {"C": 1}, } } - assert merged_information.externalAttributions == { + assert merged_information.external_attributions == { "project-id-SPDXRef-Package": opossum_package, "test-data-id-SPDXRef-File": opossum_package, } - assert merged_information.resourcesToAttributions == { + assert merged_information.resources_to_attributions == { "/A/B/": ["project-id-SPDXRef-Package"], "/A/D/C": ["test-data-id-SPDXRef-File"], } @@ -190,21 +190,21 @@ def test_expand_opossum_package_identifier() -> None: opossum_information_expanded = expand_opossum_package_identifier( OpossumInformation( metadata=Metadata( - projectId="project-id", - fileCreationDate="2022-03-02", - projectTitle="project title", + project_id="project-id", + file_creation_date="2022-03-02", + project_title="project title", ), resources={}, - externalAttributions={"SPDXRef-Package": opossum_package}, - resourcesToAttributions={"/path/to/resource": ["SPDXRef-Package"]}, + external_attributions={"SPDXRef-Package": opossum_package}, + resources_to_attributions={"/path/to/resource": ["SPDXRef-Package"]}, attributionBreakpoints=[], externalAttributionSources={}, ) ) - assert opossum_information_expanded.resourcesToAttributions == { + assert opossum_information_expanded.resources_to_attributions == { "/path/to/resource": ["project-id-SPDXRef-Package"] } - assert opossum_information_expanded.externalAttributions == { + assert opossum_information_expanded.external_attributions == { "project-id-SPDXRef-Package": opossum_package } diff --git a/tests/test_spdx/test_extract_opossum_information_from_spdx.py b/tests/test_spdx/test_extract_opossum_information_from_spdx.py index 30f328f..86aa07f 100644 --- a/tests/test_spdx/test_extract_opossum_information_from_spdx.py +++ b/tests/test_spdx/test_extract_opossum_information_from_spdx.py @@ -44,14 +44,14 @@ def test_different_paths_graph() -> None: file_tree = opossum_information.resources assert file_tree == expected_file_tree TestCase().assertCountEqual( - opossum_information.attributionBreakpoints, + opossum_information.attribution_breakpoints, [ "/SPDX Lite Document/DESCRIBES/", "/SPDX Lite Document/DESCRIBES/Example package A/CONTAINS/", "/SPDX Lite Document/DESCRIBES/Example package B/CONTAINS/", ], ) - assert opossum_information.resourcesToAttributions == { + assert opossum_information.resources_to_attributions == { "/SPDX Lite Document/": ["SPDXRef-DOCUMENT"], "/SPDX Lite Document/DESCRIBES/Example package A/": ["SPDXRef-Package-A"], "/SPDX Lite Document/DESCRIBES/Example package A/CONTAINS/Example file": [ @@ -64,7 +64,7 @@ def test_different_paths_graph() -> None: } TestCase().assertCountEqual( - opossum_information.externalAttributions.keys(), + opossum_information.external_attributions.keys(), [ "SPDXRef-DOCUMENT", "SPDXRef-Package-A", @@ -73,7 +73,7 @@ def test_different_paths_graph() -> None: ], ) - assert opossum_information.externalAttributionSources == { + assert opossum_information.external_attribution_sources == { SPDX_FILE_IDENTIFIER: ExternalAttributionSource( name=SPDX_FILE_IDENTIFIER, priority=500 ), @@ -110,7 +110,7 @@ def test_unconnected_paths_graph() -> None: file_tree = opossum_information.resources assert file_tree == expected_file_tree TestCase().assertCountEqual( - opossum_information.attributionBreakpoints, + opossum_information.attribution_breakpoints, [ "/SPDX Lite Document/DESCRIBES/", "/SPDX Lite Document/DESCRIBES/Example package A/CONTAINS/", @@ -118,7 +118,7 @@ def test_unconnected_paths_graph() -> None: ], ) - assert opossum_information.resourcesToAttributions == { + assert opossum_information.resources_to_attributions == { "/SPDX Lite Document/": ["SPDXRef-DOCUMENT"], "/SPDX Lite Document/DESCRIBES/Example package A/": ["SPDXRef-Package-A"], "/SPDX Lite Document/DESCRIBES/Example package A/CONTAINS/Example file": [ @@ -132,7 +132,7 @@ def test_unconnected_paths_graph() -> None: } TestCase().assertCountEqual( - opossum_information.externalAttributions.keys(), + opossum_information.external_attributions.keys(), [ "SPDXRef-DOCUMENT", "SPDXRef-Package-A", @@ -171,7 +171,7 @@ def test_different_roots_graph() -> None: file_tree = opossum_information.resources assert file_tree == expected_file_tree TestCase().assertCountEqual( - opossum_information.attributionBreakpoints, + opossum_information.attribution_breakpoints, [ "/Document/DESCRIBES/", "/Document/DESCRIBES/Package-A/CONTAINS/", @@ -179,7 +179,7 @@ def test_different_roots_graph() -> None: ], ) - assert opossum_information.resourcesToAttributions == { + assert opossum_information.resources_to_attributions == { "/File-B/": ["SPDXRef-File-B"], "/File-B/DESCRIBES/Package-B": ["SPDXRef-Package-B"], "/Document/": ["SPDXRef-DOCUMENT"], @@ -189,7 +189,7 @@ def test_different_roots_graph() -> None: } TestCase().assertCountEqual( - opossum_information.externalAttributions.keys(), + opossum_information.external_attributions.keys(), [ "SPDXRef-DOCUMENT", "SPDXRef-Package-A", @@ -216,7 +216,7 @@ def test_tree_generation_for_bigger_examples_json() -> None: assert len(file_tree.keys()) == 3 for attribution_breakpoint in expected_breakpoints: - assert attribution_breakpoint in opossum_information.attributionBreakpoints + assert attribution_breakpoint in opossum_information.attribution_breakpoints assert ( get_value_at_file_tree_path( file_tree, @@ -248,7 +248,7 @@ def test_tree_generation_for_bigger_examples_spdx() -> None: assert len(file_tree.keys()) == 2 for attribution_breakpoint in expected_breakpoints: - assert attribution_breakpoint in opossum_information.attributionBreakpoints + assert attribution_breakpoint in opossum_information.attribution_breakpoints assert ( get_value_at_file_tree_path( From 49e6fe649faf1f9f6282901f887597d087589438 Mon Sep 17 00:00:00 2001 From: Adrian Braemer Date: Thu, 16 Jan 2025 14:34:01 +0100 Subject: [PATCH 2/2] ci: enable ruff linter to check for snake_case --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a2e17b1..982646a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,8 @@ select = [ "SIM", # isort "I", + # naming + "N" ] extend-select = ["E501"]