From 699b46cd57d659312647a8ac194279d1ccf0c8ab Mon Sep 17 00:00:00 2001 From: Nelson Moore Date: Tue, 8 Aug 2023 10:19:18 -0400 Subject: [PATCH] refactor: move shared changelog utilities - Move functions shared by multiple changelog scripts to util/changelog.py from the individual scripts - Reset changeset id for changelog tests --- python/scripts/make_diff_changelog.py | 35 ++-------------------- python/scripts/make_mapping_changelog.py | 36 ++-------------------- python/scripts/make_model_changelog.py | 32 ++------------------ python/src/bento_meta/objects.py | 1 + python/src/bento_meta/util/__init__.py | 3 +- python/src/bento_meta/util/changelog.py | 38 ++++++++++++++++++++++++ python/tests/samples/test_changelog.ini | 2 +- python/tests/test_014changelog.py | 7 +++-- 8 files changed, 53 insertions(+), 101 deletions(-) create mode 100644 python/src/bento_meta/util/changelog.py diff --git a/python/scripts/make_diff_changelog.py b/python/scripts/make_diff_changelog.py index 6ccef7f..20f23c1 100644 --- a/python/scripts/make_diff_changelog.py +++ b/python/scripts/make_diff_changelog.py @@ -4,16 +4,16 @@ an MDB in Neo4J to update the model from the old version to the new one. """ -import configparser import logging from pathlib import Path -from typing import Dict, Generator, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import click from bento_mdf.diff import diff_models from bento_mdf.mdf import MDF from bento_meta.entity import Entity from bento_meta.objects import Edge, Node, Property, Term +from bento_meta.util.changelog import changeset_id_generator, update_config_changeset_id from bento_meta.util.cypher.clauses import ( Delete, DetachDelete, @@ -291,37 +291,6 @@ def convert_diff_segment_to_cypher_statement( return stmt -def get_initial_changeset_id(config_file_path: str) -> int: - """Gets initial changeset id from changelog config file""" - config = configparser.ConfigParser() - config.read(config_file_path) - try: - return config.getint(section="changelog", option="changeset_id") - except (configparser.NoSectionError, configparser.NoOptionError) as error: - print(error) - raise - - -def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: - """ - Iterator for changeset_id. Gets latest changeset id from changelog.ini - and provides up-to-date changeset id as - """ - i = get_initial_changeset_id(config_file_path) - while True: - yield i - i += 1 - - -def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: - """Updates changelog config file with new changeset id""" - config = configparser.ConfigParser() - config.read(config_file_path) - config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) - with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: - config.write(fp=config_file) - - def convert_diff_to_changelog( diff, model_handle: str, author: str, config_file_path: str ) -> Changelog: diff --git a/python/scripts/make_mapping_changelog.py b/python/scripts/make_mapping_changelog.py index cb20075..0abb70f 100644 --- a/python/scripts/make_mapping_changelog.py +++ b/python/scripts/make_mapping_changelog.py @@ -1,11 +1,12 @@ """converts mapping MDF to changelog""" -import configparser + from ast import literal_eval from pathlib import Path -from typing import Dict, Generator, List, Optional, Union +from typing import Dict, List, Optional, Union import click import yaml +from bento_meta.util.changelog import changeset_id_generator, update_config_changeset_id from bento_meta.util.cypher.clauses import ( Case, Create, @@ -21,37 +22,6 @@ from liquichange.changelog import Changelog, Changeset, CypherChange -def get_initial_changeset_id(config_file_path: str) -> int: - """Gets initial changeset id from changelog config file""" - config = configparser.ConfigParser() - config.read(config_file_path) - try: - return config.getint(section="changelog", option="changeset_id") - except (configparser.NoSectionError, configparser.NoOptionError) as error: - print(error) - raise - - -def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: - """ - Iterator for changeset_id. Gets latest changeset id from changelog.ini - and provides up-to-date changeset id as - """ - i = get_initial_changeset_id(config_file_path) - while True: - yield i - i += 1 - - -def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: - """Updates changelog config file with new changeset id""" - config = configparser.ConfigParser() - config.read(config_file_path) - config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) - with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: - config.write(fp=config_file) - - def load_yaml( file: str, ) -> Dict[ diff --git a/python/scripts/make_model_changelog.py b/python/scripts/make_model_changelog.py index f6bc237..8d59af3 100644 --- a/python/scripts/make_model_changelog.py +++ b/python/scripts/make_model_changelog.py @@ -2,10 +2,9 @@ Script takes one MDF file representing a model and produces a Liquibase Changelog with the necessary changes to add the model to an MDB """ -import configparser import logging from pathlib import Path -from typing import Dict, Generator, List, Optional, Union +from typing import Dict, List, Optional, Union import click from bento_mdf.mdf import MDF @@ -13,6 +12,7 @@ from bento_meta.mdb.mdb import make_nanoid from bento_meta.model import Model from bento_meta.objects import Concept, Term, ValueSet +from bento_meta.util.changelog import changeset_id_generator, update_config_changeset_id from bento_meta.util.cypher.clauses import Create, Match, Merge, OnCreateSet, Statement from bento_meta.util.cypher.entities import N, R, T, _plain_var from liquichange.changelog import Changelog, Changeset, CypherChange @@ -20,34 +20,6 @@ logger = logging.getLogger(__name__) -def get_initial_changeset_id(config_file_path: str) -> int: - """Gets initial changeset id from changelog config file""" - config = configparser.ConfigParser() - config.read(config_file_path) - try: - return config.getint(section="changelog", option="changeset_id") - except (configparser.NoSectionError, configparser.NoOptionError) as error: - logger.error(f"Reading changeset ID failed: {error}") - raise - - -def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: - """Generates sequential changeset IDs by reading the latest ID from a config file.""" - i = get_initial_changeset_id(config_file_path) - while True: - yield i - i += 1 - - -def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: - """Updates changelog config file with new changeset id.""" - config = configparser.ConfigParser() - config.read(config_file_path) - config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) - with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: - config.write(fp=config_file) - - def escape_quotes_in_attr(entity: Entity) -> None: """ Escapes quotes in entity attributes. diff --git a/python/src/bento_meta/objects.py b/python/src/bento_meta/objects.py index e972752..5d02987 100644 --- a/python/src/bento_meta/objects.py +++ b/python/src/bento_meta/objects.py @@ -91,6 +91,7 @@ class Property(Entity): "units": "units", "item_domain": "item_domain", "is_required": "is_required", + "parent_handle": "parent_handle", }, "relationship": { "concept": {"rel": ":has_concept>", "end_cls": "Concept"}, diff --git a/python/src/bento_meta/util/__init__.py b/python/src/bento_meta/util/__init__.py index 74367c4..21a652f 100644 --- a/python/src/bento_meta/util/__init__.py +++ b/python/src/bento_meta/util/__init__.py @@ -3,6 +3,7 @@ bento_meta.util.cypher - Represent Cypher queries programmatically bento_meta.util.makeq - Create Cypher queries from API endpoint paths +bento_meta.util.changelog - Utilities for working with Liquibase Changelogs """ -from . import cypher +from . import changelog, cypher diff --git a/python/src/bento_meta/util/changelog.py b/python/src/bento_meta/util/changelog.py new file mode 100644 index 0000000..ac90695 --- /dev/null +++ b/python/src/bento_meta/util/changelog.py @@ -0,0 +1,38 @@ +""" +bento_meta.util.changelog + +Common functions shared by changelog generation scripts +""" +import configparser +import logging +from typing import Generator, Optional + +logger = logging.getLogger(__name__) + + +def get_initial_changeset_id(config_file_path: str) -> int: + """Gets initial changeset id from changelog config file""" + config = configparser.ConfigParser() + config.read(config_file_path) + try: + return config.getint(section="changelog", option="changeset_id") + except (configparser.NoSectionError, configparser.NoOptionError) as error: + logger.error(f"Reading changeset ID failed: {error}") + raise + + +def changeset_id_generator(config_file_path: str) -> Generator[int, None, None]: + """Generates sequential changeset IDs by reading the latest ID from a config file.""" + i = get_initial_changeset_id(config_file_path) + while True: + yield i + i += 1 + + +def update_config_changeset_id(config_file_path: str, new_changeset_id: int) -> None: + """Updates changelog config file with new changeset id.""" + config = configparser.ConfigParser() + config.read(config_file_path) + config.set(section="changelog", option="changeset_id", value=str(new_changeset_id)) + with open(file=config_file_path, mode="w", encoding="UTF-8") as config_file: + config.write(fp=config_file) diff --git a/python/tests/samples/test_changelog.ini b/python/tests/samples/test_changelog.ini index 5db2710..69baef7 100644 --- a/python/tests/samples/test_changelog.ini +++ b/python/tests/samples/test_changelog.ini @@ -1,3 +1,3 @@ [changelog] -changeset_id = 6855 +changeset_id = 1 diff --git a/python/tests/test_014changelog.py b/python/tests/test_014changelog.py index 015a78a..9763285 100644 --- a/python/tests/test_014changelog.py +++ b/python/tests/test_014changelog.py @@ -5,6 +5,7 @@ from bento_mdf.diff import diff_models from bento_mdf.mdf import MDF from bento_meta.objects import Property +from bento_meta.util.changelog import update_config_changeset_id from scripts.make_diff_changelog import convert_diff_to_changelog from scripts.make_mapping_changelog import convert_mappings_to_changelog from scripts.make_model_changelog import ( @@ -33,6 +34,7 @@ def test_make_model_changelog(): changelog = convert_model_to_changelog( model=mdf.model, author=AUTHOR, config_file_path=TEST_CHANGELOG_CONFIG ) + update_config_changeset_id(TEST_CHANGELOG_CONFIG, 1) assert len(changelog.subelements) == 46 @@ -47,6 +49,7 @@ def test_make_diff_changelog(): author=AUTHOR, config_file_path=TEST_CHANGELOG_CONFIG, ) + update_config_changeset_id(TEST_CHANGELOG_CONFIG, 1) assert len(changelog.subelements) == 33 @@ -58,6 +61,7 @@ def test_make_mapping_changelog(): config_file_path=TEST_CHANGELOG_CONFIG, _commit=_COMMIT, ) + update_config_changeset_id(TEST_CHANGELOG_CONFIG, 1) assert len(changelog.subelements) == 6 @@ -69,6 +73,3 @@ def test_escape_quotes_in_attr(): assert prop.handle == r"""Quote\'s Handle""" assert prop.desc == r"""quote\'s quote\'s \"quotes\"""" - - -test_make_model_changelog()