From 53873812cfc5efe49af90406ce636a45b8cb7315 Mon Sep 17 00:00:00 2001 From: Nelson Moore Date: Mon, 7 Aug 2023 12:35:35 -0400 Subject: [PATCH 1/4] fix: update property attrs Add nanoid to Property entities in make_model_changlog.py script to prevent the below in generated changelogs: - properties being "reused" by multiple parent nodes/edges - duplicate relationships between the prop and connected value sets & parent entities Also add "parent_handle" attribute to Property entity in bento_meta objects for the same reasons - should allow Property entities to contain all the info required to ensure they are added properly to MDB w/o requiring additional entities See property uniqueness/distinctness conventions in https://cbiit.github.io/bento-meta/mdb-conventions.html --- python/scripts/make_model_changelog.py | 1 + python/src/bento_meta/objects.py | 1 + 2 files changed, 2 insertions(+) diff --git a/python/scripts/make_model_changelog.py b/python/scripts/make_model_changelog.py index 49fdadf..f6bc237 100644 --- a/python/scripts/make_model_changelog.py +++ b/python/scripts/make_model_changelog.py @@ -191,6 +191,7 @@ def process_props(entity: Entity, cypher_stmts) -> None: if not entity.props: return for prop in entity.props.values(): + prop.nanoid = make_nanoid() generate_cypher_to_add_entity(prop, cypher_stmts) generate_cypher_to_add_relationship(entity, "has_property", prop, cypher_stmts) process_tags(prop, cypher_stmts) diff --git a/python/src/bento_meta/objects.py b/python/src/bento_meta/objects.py index 14be94e..e972752 100644 --- a/python/src/bento_meta/objects.py +++ b/python/src/bento_meta/objects.py @@ -77,6 +77,7 @@ class Property(Entity): "is_required": "simple", "concept": "object", "value_set": "object", + "parent_handle": "simple", } mapspec_ = { "label": "property", From 699b46cd57d659312647a8ac194279d1ccf0c8ab Mon Sep 17 00:00:00 2001 From: Nelson Moore Date: Tue, 8 Aug 2023 10:19:18 -0400 Subject: [PATCH 2/4] refactor: move shared changelog utilities - Move functions shared by multiple changelog scripts to util/changelog.py from the individual scripts - Reset changeset id for changelog tests --- python/scripts/make_diff_changelog.py | 35 ++-------------------- python/scripts/make_mapping_changelog.py | 36 ++-------------------- python/scripts/make_model_changelog.py | 32 ++------------------ python/src/bento_meta/objects.py | 1 + python/src/bento_meta/util/__init__.py | 3 +- python/src/bento_meta/util/changelog.py | 38 ++++++++++++++++++++++++ python/tests/samples/test_changelog.ini | 2 +- python/tests/test_014changelog.py | 7 +++-- 8 files changed, 53 insertions(+), 101 deletions(-) create mode 100644 python/src/bento_meta/util/changelog.py diff --git a/python/scripts/make_diff_changelog.py b/python/scripts/make_diff_changelog.py index 6ccef7f..20f23c1 100644 --- a/python/scripts/make_diff_changelog.py +++ b/python/scripts/make_diff_changelog.py @@ -4,16 +4,16 @@ an MDB in Neo4J to update the model from the old version to the new one. """ -import configparser import logging from pathlib import Path -from typing import Dict, Generator, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import click from bento_mdf.diff import diff_models from bento_mdf.mdf import MDF from bento_meta.entity import Entity from bento_meta.objects import Edge, Node, Property, Term +from bento_meta.util.changelog import changeset_id_generator, update_config_changeset_id from bento_meta.util.cypher.clauses import ( Delete, DetachDelete, @@ -291,37 +291,6 @@ def convert_diff_segment_to_cypher_statement( return stmt -def get_initial_changeset_id(config_file_path: str) -> int: - """Gets initial changeset id from changelog config file""" - config = configparser.ConfigParser() - config.read(config_file_path) - try: - return config.getint(section="changelog", option="changeset_id") - except (configparser.NoSectionError, configparser.NoOptionError) as error: - print(error) - raise - - -def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: - """ - Iterator for changeset_id. Gets latest changeset id from changelog.ini - and provides up-to-date changeset id as - """ - i = get_initial_changeset_id(config_file_path) - while True: - yield i - i += 1 - - -def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: - """Updates changelog config file with new changeset id""" - config = configparser.ConfigParser() - config.read(config_file_path) - config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) - with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: - config.write(fp=config_file) - - def convert_diff_to_changelog( diff, model_handle: str, author: str, config_file_path: str ) -> Changelog: diff --git a/python/scripts/make_mapping_changelog.py b/python/scripts/make_mapping_changelog.py index cb20075..0abb70f 100644 --- a/python/scripts/make_mapping_changelog.py +++ b/python/scripts/make_mapping_changelog.py @@ -1,11 +1,12 @@ """converts mapping MDF to changelog""" -import configparser + from ast import literal_eval from pathlib import Path -from typing import Dict, Generator, List, Optional, Union +from typing import Dict, List, Optional, Union import click import yaml +from bento_meta.util.changelog import changeset_id_generator, update_config_changeset_id from bento_meta.util.cypher.clauses import ( Case, Create, @@ -21,37 +22,6 @@ from liquichange.changelog import Changelog, Changeset, CypherChange -def get_initial_changeset_id(config_file_path: str) -> int: - """Gets initial changeset id from changelog config file""" - config = configparser.ConfigParser() - config.read(config_file_path) - try: - return config.getint(section="changelog", option="changeset_id") - except (configparser.NoSectionError, configparser.NoOptionError) as error: - print(error) - raise - - -def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: - """ - Iterator for changeset_id. Gets latest changeset id from changelog.ini - and provides up-to-date changeset id as - """ - i = get_initial_changeset_id(config_file_path) - while True: - yield i - i += 1 - - -def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: - """Updates changelog config file with new changeset id""" - config = configparser.ConfigParser() - config.read(config_file_path) - config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) - with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: - config.write(fp=config_file) - - def load_yaml( file: str, ) -> Dict[ diff --git a/python/scripts/make_model_changelog.py b/python/scripts/make_model_changelog.py index f6bc237..8d59af3 100644 --- a/python/scripts/make_model_changelog.py +++ b/python/scripts/make_model_changelog.py @@ -2,10 +2,9 @@ Script takes one MDF file representing a model and produces a Liquibase Changelog with the necessary changes to add the model to an MDB """ -import configparser import logging from pathlib import Path -from typing import Dict, Generator, List, Optional, Union +from typing import Dict, List, Optional, Union import click from bento_mdf.mdf import MDF @@ -13,6 +12,7 @@ from bento_meta.mdb.mdb import make_nanoid from bento_meta.model import Model from bento_meta.objects import Concept, Term, ValueSet +from bento_meta.util.changelog import changeset_id_generator, update_config_changeset_id from bento_meta.util.cypher.clauses import Create, Match, Merge, OnCreateSet, Statement from bento_meta.util.cypher.entities import N, R, T, _plain_var from liquichange.changelog import Changelog, Changeset, CypherChange @@ -20,34 +20,6 @@ logger = logging.getLogger(__name__) -def get_initial_changeset_id(config_file_path: str) -> int: - """Gets initial changeset id from changelog config file""" - config = configparser.ConfigParser() - config.read(config_file_path) - try: - return config.getint(section="changelog", option="changeset_id") - except (configparser.NoSectionError, configparser.NoOptionError) as error: - logger.error(f"Reading changeset ID failed: {error}") - raise - - -def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: - """Generates sequential changeset IDs by reading the latest ID from a config file.""" - i = get_initial_changeset_id(config_file_path) - while True: - yield i - i += 1 - - -def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: - """Updates changelog config file with new changeset id.""" - config = configparser.ConfigParser() - config.read(config_file_path) - config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) - with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: - config.write(fp=config_file) - - def escape_quotes_in_attr(entity: Entity) -> None: """ Escapes quotes in entity attributes. diff --git a/python/src/bento_meta/objects.py b/python/src/bento_meta/objects.py index e972752..5d02987 100644 --- a/python/src/bento_meta/objects.py +++ b/python/src/bento_meta/objects.py @@ -91,6 +91,7 @@ class Property(Entity): "units": "units", "item_domain": "item_domain", "is_required": "is_required", + "parent_handle": "parent_handle", }, "relationship": { "concept": {"rel": ":has_concept>", "end_cls": "Concept"}, diff --git a/python/src/bento_meta/util/__init__.py b/python/src/bento_meta/util/__init__.py index 74367c4..21a652f 100644 --- a/python/src/bento_meta/util/__init__.py +++ b/python/src/bento_meta/util/__init__.py @@ -3,6 +3,7 @@ bento_meta.util.cypher - Represent Cypher queries programmatically bento_meta.util.makeq - Create Cypher queries from API endpoint paths +bento_meta.util.changelog - Utilities for working with Liquibase Changelogs """ -from . import cypher +from . import changelog, cypher diff --git a/python/src/bento_meta/util/changelog.py b/python/src/bento_meta/util/changelog.py new file mode 100644 index 0000000..ac90695 --- /dev/null +++ b/python/src/bento_meta/util/changelog.py @@ -0,0 +1,38 @@ +""" +bento_meta.util.changelog + +Common functions shared by changelog generation scripts +""" +import configparser +import logging +from typing import Generator, Optional + +logger = logging.getLogger(__name__) + + +def get_initial_changeset_id(config_file_path: str) -> int: + """Gets initial changeset id from changelog config file""" + config = configparser.ConfigParser() + config.read(config_file_path) + try: + return config.getint(section="changelog", option="changeset_id") + except (configparser.NoSectionError, configparser.NoOptionError) as error: + logger.error(f"Reading changeset ID failed: {error}") + raise + + +def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: + """Generates sequential changeset IDs by reading the latest ID from a config file.""" + i = get_initial_changeset_id(config_file_path) + while True: + yield i + i += 1 + + +def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: + """Updates changelog config file with new changeset id.""" + config = configparser.ConfigParser() + config.read(config_file_path) + config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) + with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: + config.write(fp=config_file) diff --git a/python/tests/samples/test_changelog.ini b/python/tests/samples/test_changelog.ini index 5db2710..69baef7 100644 --- a/python/tests/samples/test_changelog.ini +++ b/python/tests/samples/test_changelog.ini @@ -1,3 +1,3 @@ [changelog] -changeset_id = 6855 +changeset_id = 1 diff --git a/python/tests/test_014changelog.py b/python/tests/test_014changelog.py index 015a78a..9763285 100644 --- a/python/tests/test_014changelog.py +++ b/python/tests/test_014changelog.py @@ -5,6 +5,7 @@ from bento_mdf.diff import diff_models from bento_mdf.mdf import MDF from bento_meta.objects import Property +from bento_meta.util.changelog import update_config_changeset_id from scripts.make_diff_changelog import convert_diff_to_changelog from scripts.make_mapping_changelog import convert_mappings_to_changelog from scripts.make_model_changelog import ( @@ -33,6 +34,7 @@ def test_make_model_changelog(): changelog = convert_model_to_changelog( model=mdf.model, author=AUTHOR, config_file_path=TEST_CHANGELOG_CONFIG ) + update_config_changeset_id(TEST_CHANGELOG_CONFIG, 1) assert len(changelog.subelements) == 46 @@ -47,6 +49,7 @@ def test_make_diff_changelog(): author=AUTHOR, config_file_path=TEST_CHANGELOG_CONFIG, ) + update_config_changeset_id(TEST_CHANGELOG_CONFIG, 1) assert len(changelog.subelements) == 33 @@ -58,6 +61,7 @@ def test_make_mapping_changelog(): config_file_path=TEST_CHANGELOG_CONFIG, _commit=_COMMIT, ) + update_config_changeset_id(TEST_CHANGELOG_CONFIG, 1) assert len(changelog.subelements) == 6 @@ -69,6 +73,3 @@ def test_escape_quotes_in_attr(): assert prop.handle == r"""Quote\'s Handle""" assert prop.desc == r"""quote\'s quote\'s \"quotes\"""" - - -test_make_model_changelog() From 36804cc6e2af892f45060e29acab26edaeed2b1e Mon Sep 17 00:00:00 2001 From: Nelson Moore Date: Tue, 29 Aug 2023 10:41:37 -0400 Subject: [PATCH 3/4] bugfix: update make_model_changelog.py edges Add nanoid to Edge entities in make_model_changlog.py script to prevent duplicate relationships from edges with identical handles to src/dst Nodes --- python/scripts/make_model_changelog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/scripts/make_model_changelog.py b/python/scripts/make_model_changelog.py index 8d59af3..c51ad04 100644 --- a/python/scripts/make_model_changelog.py +++ b/python/scripts/make_model_changelog.py @@ -183,6 +183,7 @@ def process_model_nodes(model: Model, cypher_stmts) -> None: def process_model_edges(model: Model, cypher_stmts) -> None: """Generates cypher statements to create/merge an model's edges.""" for edge in model.edges.values(): + edge.nanoid = make_nanoid() generate_cypher_to_add_entity(edge, cypher_stmts) generate_cypher_to_add_relationship(edge, "has_src", edge.src, cypher_stmts) generate_cypher_to_add_relationship(edge, "has_dst", edge.dst, cypher_stmts) From c8f0d07ef8e10a9028df89e157938c9c8953d9bd Mon Sep 17 00:00:00 2001 From: Nelson Moore Date: Tue, 29 Aug 2023 10:43:55 -0400 Subject: [PATCH 4/4] chore: bump version Bump version in pyproject.toml from v0.2.3 to v0.2.4 --- python/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9f13936..c205216 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bento-meta" -version = "0.2.3" +version = "0.2.4" description = "Python drivers for Bento Metamodel Database" authors = [ { name="Mark A. Jensen", email = "mark.jensen@nih.gov"}, @@ -20,7 +20,7 @@ classifiers = [ [tool.poetry] name = "bento-meta" -version = "0.2.3" +version = "0.2.4" description = "Python drivers for Bento Metamodel Database" authors = [ "Mark A. Jensen ",