From f2560c61aefc19b090620c5b8cb20dbb278d7a94 Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Mon, 10 Aug 2020 08:36:36 +0200 Subject: [PATCH 01/22] Tool to convert YAML ontologies with ALL_CAPS entity names to CamelCase (#459) Tool to convert YAML ontologies with ALL_CAPS entity names to CamelCase (#459) --- osp/core/ontology/docs/city.ontology.yml | 6 +- osp/core/ontology/yml/yml_validator.py | 2 +- osp/core/tools/yaml2camelcase.py | 188 ++++++++++++++++++ setup.py | 3 +- tests/city_caps.ontology.yml | 243 +++++++++++++++++++++++ tests/test_yaml2camelcase.py | 34 ++++ 6 files changed, 469 insertions(+), 7 deletions(-) create mode 100644 osp/core/tools/yaml2camelcase.py create mode 100644 tests/city_caps.ontology.yml create mode 100644 tests/test_yaml2camelcase.py diff --git a/osp/core/ontology/docs/city.ontology.yml b/osp/core/ontology/docs/city.ontology.yml index 79b3d679..e74fcbb9 100644 --- a/osp/core/ontology/docs/city.ontology.yml +++ b/osp/core/ontology/docs/city.ontology.yml @@ -202,7 +202,7 @@ ontology: subclass_of: - city.ArchitecturalStructure - city.hasPart: - range: city.Adress + range: city.Address cardinality: 1 exclusive: false - city.hasPart: @@ -233,10 +233,6 @@ ontology: Citizen: subclass_of: - city.Person - - city.isMajorOf: - range: city.City - cardinality: 0-1 - exclusive: true age: subclass_of: diff --git a/osp/core/ontology/yml/yml_validator.py b/osp/core/ontology/yml/yml_validator.py index 84b9b300..4fe60374 100644 --- a/osp/core/ontology/yml/yml_validator.py +++ b/osp/core/ontology/yml/yml_validator.py @@ -17,7 +17,7 @@ namespace_name_pattern = re.compile(namespace_name_regex) entity_name_pattern = re.compile(r"^%s$" % entity_name_regex) qualified_entity_name_pattern = re.compile( - r"^%s.%s$" % (namespace_name_regex, entity_name_regex)) + r"^%s\.%s$" % (namespace_name_regex, entity_name_regex)) entity_common_keys = { DESCRIPTION_KEY: str, diff --git a/osp/core/tools/yaml2camelcase.py b/osp/core/tools/yaml2camelcase.py new file mode 100644 index 00000000..e3e663ff --- /dev/null +++ b/osp/core/tools/yaml2camelcase.py @@ -0,0 +1,188 @@ +import argparse +import yaml +import logging +import re +import shutil +from copy import deepcopy +from pathlib import Path +from osp.core.ontology.yml.yml_parser import YmlParser +from osp.core.ontology.yml.yml_keywords import NAMESPACE_KEY, ONTOLOGY_KEY, \ + SUPERCLASSES_KEY + + +entity_name_regex = r"(_|[A-Z])([A-Z]|[0-9]|_)*" +entity_name_pattern = re.compile(r"^%s$" % entity_name_regex) +qualified_entity_name_pattern = re.compile( + r"^%s\.%s$" % tuple([entity_name_regex] * 2) +) + +logger = logging.getLogger(__name__) + + +class Yaml2CamelCaseConverter(): + """Tool that transforms a YAML ontologies with entity name in ALL_CAPS + to a YAML ontology in CamelCase. + """ + + def __init__(self, file_path): + """Initialize the converter + + Args: + file_path (path): Path to the yaml file to convert + """ + self.file_path = file_path + self.doc = YmlParser.get_doc(self.file_path) + self.onto_doc = self.doc[ONTOLOGY_KEY] + self.orig_onto_doc = deepcopy(self.onto_doc) + self.namespace = self.doc[NAMESPACE_KEY].lower() + + def convert(self): + """Convert the yaml file to CamelCase""" + self.doc[NAMESPACE_KEY] = self.namespace + self.convert_nested_doc(self.onto_doc, pattern=entity_name_pattern) + + def convert_nested_doc(self, doc, pattern=qualified_entity_name_pattern): + """Convert the document to CamelCase + + Args: + doc (Any): The document to convert + pattern (re.Pattern, optional): The pattern for the entities. + Defaults to qualified_entity_name_pattern. + """ + if isinstance(doc, list): + new = list() + for elem in doc: + if elem == "CUBA.ENTITY": + new.append("cuba.Class") + elif isinstance(elem, str) and pattern.match(elem): + new.append(self.toCamelCase(elem)) + else: + new.append(elem) + self.convert_nested_doc(elem) + doc.clear() + doc.extend(new) + + if isinstance(doc, dict): + new = dict() + for key, value in doc.items(): + if isinstance(key, str) and pattern.match(key): + new[self.toCamelCase(key)] = value + self.convert_nested_doc(value) + elif value == "CUBA.ENTITY": + new[key] = "cuba.Class" + elif isinstance(value, str) and pattern.match(value): + new[key] = self.toCamelCase(value) + else: + new[key] = value + self.convert_nested_doc(value) + doc.clear() + doc.update(new) + + def store(self, output): + """Update the ontology on disc""" + if not output: + output = self.file_path + if output == self.file_path: + logger.info(f"Backing up original file at {output}.orig") + shutil.copyfile(str(output), str(output) + ".orig") + + logger.info(f"Writing camel case file to {output}") + with open(output, "w") as f: + yaml.safe_dump(self.doc, f) + + def get_first_letter_caps(self, word, internal=False): + """Check whether a entity name should start with lower or uppercase + + Args: + word (str): The entity name to check + internal (bool, optional): True iff this method has been + called recursively. Defaults to False. + + Raises: + ValueError: Undefined entity name + + Returns: + Optional[bool]: True iff word should start with uppercase. + Will always return a bool if internal is False. + """ + # cuba cases + if word in ["CUBA.RELATIONSHIP", "CUBA.ACTIVE_RELATIONSHIP", + "CUBA.PASSIVE_RELATIONSHIP", "CUBA.ATTRIBUTE", + "CUBA.PATH"]: + return False + if word in ["CUBA.WRAPPER", "CUBA.NOTHING", "CUBA.FILE"]: + return True + if word == "CUBA.ENTITY": + return None if internal else True + + # qualified cases + if "." in word: + namespace, name = word.split(".") + if namespace.lower() == self.namespace: + x = self.get_first_letter_caps(name, internal=True) + return True if x is None and not internal else x + return input(f"Is {word} a ontology class?") \ + .lower().strip().startswith("y") + + # unqualified cases + if word not in self.orig_onto_doc: + raise ValueError(f"Undefined entity {word}") + + subclasses = self.orig_onto_doc[word][SUPERCLASSES_KEY] + if any(isinstance(subclass, dict) for subclass in subclasses): + return True + if any(self.get_first_letter_caps(subclass, True) is False + for subclass in subclasses): + return False + if any(self.get_first_letter_caps(subclass, True) is True + for subclass in subclasses): + return True + + return None if internal else True + + def toCamelCase(self, word): + """Convert the given entity name to camel case. + + Args: + word (str): The word to convert to CamelCase + + Returns: + str: The entity name in CamelCase + """ + first_letter_caps = self.get_first_letter_caps(word) + result = "" + if "." in word: + result += word.split(".")[0].lower() + "." + word = word.split(".")[1] + result += word[0].upper() if first_letter_caps else word[0].lower() + next_upper = False + for c in word[1:]: + if c == "_": + next_upper = True + elif next_upper: + result += c.upper() + next_upper = False + else: + result += c.lower() + return result + + +def run_from_terminal(): + """Run yaml2camelcase from the terminal""" + # Parse the user arguments + parser = argparse.ArgumentParser( + description="Convert a YAML ontology with ALL_CAPS entity names to a " + "YAML ontology using CamelCase" + ) + parser.add_argument("input", type=Path, help="The input yaml file.") + parser.add_argument("-o", "--output", type=Path, required=False, + default=None, help="The input yaml file.") + args = parser.parse_args() + + c = Yaml2CamelCaseConverter(args.input) + c.convert() + c.store(args.output) + + +if __name__ == "__main__": + run_from_terminal() diff --git a/setup.py b/setup.py index a87f0c64..2ca8fdb3 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,8 @@ 'console_scripts': { 'owl2yml = osp.core.tools.owl2yml:run_from_terminal', 'pico = osp.core.pico:install_from_terminal', - 'ontology2dot = osp.core.tools.ontology2dot:run_from_terminal' + 'ontology2dot = osp.core.tools.ontology2dot:run_from_terminal', + 'yaml2camelcase = osp.core.tools.yaml2camelcase:run_from_terminal' } }, install_requires=[ diff --git a/tests/city_caps.ontology.yml b/tests/city_caps.ontology.yml new file mode 100644 index 00000000..095b020d --- /dev/null +++ b/tests/city_caps.ontology.yml @@ -0,0 +1,243 @@ +--- +version: "0.0.3" + +namespace: "CITY" + +ontology: + + ################ + + ENCLOSES: + subclass_of: + - CUBA.ACTIVE_RELATIONSHIP + inverse: CITY.IS_ENCLOSED_BY + + HAS_PART: + subclass_of: + - CITY.ENCLOSES + inverse: CITY.IS_PART_OF + default_rel: true + characteristics: + - transitive + + IS_ENCLOSED_BY: + subclass_of: + - CUBA.PASSIVE_RELATIONSHIP + inverse: CITY.ENCLOSES + + IS_PART_OF: + subclass_of: + - CITY.IS_ENCLOSED_BY + inverse: CITY.HAS_PART + + HAS_INHABITANT: + subclass_of: + - CITY.ENCLOSES + + HAS_CHILD: + subclass_of: + - CITY.HAS_PART + inverse: CITY.IS_CHILD_OF + + IS_CHILD_OF: + subclass_of: + - CITY.IS_PART_OF + inverse: CITY.HAS_CHILD + + HAS_WORKER: + subclass_of: + - CITY.HAS_PART + inverse: CITY.WORKS_IN + + WORKS_IN: + subclass_of: + - CITY.IS_PART_OF + inverse: CITY.HAS_WORKER + + HAS_MAJOR: + subclass_of: + - CITY.HAS_WORKER + + ################ + + CITY_WRAPPER: + subclass_of: + - CUBA.WRAPPER + - CITY.HAS_PART: + range: CITY.CITY + cardinality: 1+ + exclusive: false + + CITY_SIM_WRAPPER: + subclass_of: + - CITY.CITY_WRAPPER + - CITY.HAS_PART: + range: CITY.PERSON + cardinality: many + exclusive: false + - CITY.HAS_PART: + range: CITY.CITY + cardinality: 1 + exclusive: false + attributes: + CITY.NUM_STEPS: + + NUM_STEPS: + subclass_of: + - CITY.NUMBER + + ################ + + GEOGRAPHICAL_PLACE: + subclass_of: + - CUBA.ENTITY + attributes: + CITY.NAME: + + ARCHITECTURAL_COMPONENT: + subclass_of: + - CUBA.ENTITY + + LIVING_BEING: + description: A being that lives + subclass_of: + - CUBA.ENTITY + - CITY.IS_CHILD_OF: + range: CITY.LIVING_BEING + cardinality: 0-2 + exclusive: false + attributes: + CITY.NAME: "John Smith" + CITY.AGE: 25 + + NUMBER: + subclass_of: + - CUBA.ATTRIBUTE + datatype: INT + + POSTAL_CODE: + subclass_of: + - CITY.NUMBER + + NAME: + subclass_of: + - CUBA.ATTRIBUTE + + COORDINATES: + subclass_of: + - CUBA.ATTRIBUTE + datatype: VECTOR:INT:2 + + ADDRESS: + subclass_of: + - CUBA.ENTITY + - CITY.IS_PART_OF: + range: CITY.BUILDING + cardinality: 1 + exclusive: false + attributes: + CITY.NAME: "Street" + CITY.NUMBER: 7 + CITY.POSTAL_CODE: + + POPULATED_PLACE: + subclass_of: + - CITY.GEOGRAPHICAL_PLACE + - CITY.HAS_INHABITANT: + range: CITY.CITIZEN + cardinality: many + exclusive: true + - CITY.HAS_WORKER: + range: CITY.PERSON + cardinality: many + exclusive: true + attributes: + CITY.COORDINATES: [0, 0] + + CITY: + subclass_of: + - CITY.POPULATED_PLACE + - CITY.HAS_PART: + range: CITY.NEIGHBORHOOD + cardinality: many + exclusive: true + - CITY.IS_PART_OF: + range: CITY.CITY_WRAPPER + cardinality: 0-1 + exclusive: true + - CITY.HAS_MAJOR: + range: CITY.CITIZEN + cardinality: 0-1 + exclusive: true + + NEIGHBORHOOD: + subclass_of: + - CITY.POPULATED_PLACE + - CITY.IS_PART_OF: + range: CITY.CITY + cardinality: 1 + exclusive: true + - CITY.HAS_PART: + range: CITY.STREET + cardinality: many + exclusive: true + + STREET: + subclass_of: + - CITY.POPULATED_PLACE + - CITY.HAS_PART: + range: CITY.BUILDING + cardinality: many + exclusive: true + - CITY.IS_PART_OF: + range: CITY.NEIGHBORHOOD + cardinality: 1 + exclusive: true + + ARCHITECTURAL_STRUCTURE: + subclass_of: + - CITY.GEOGRAPHICAL_PLACE + + BUILDING: + subclass_of: + - CITY.ARCHITECTURAL_STRUCTURE + - CITY.HAS_PART: + range: CITY.ADDRESS + cardinality: 1 + exclusive: false + - CITY.HAS_PART: + range: CITY.FLOOR + cardinality: many + exclusive: false + - CITY.IS_PART_OF: + range: CITY.STREET + cardinality: 1 + exclusive: true + + FLOOR: + subclass_of: + - CITY.ARCHITECTURAL_COMPONENT + + PERSON: + subclass_of: + - CITY.LIVING_BEING + - CITY.WORKS_IN: + range: CITY.POPULATED_PLACE + cardinality: 0-1 + exclusive: true + - CITY.IS_PART_OF: + range: CITY.CITY_SIM_WRAPPER + cardinality: 0-1 + exclusive: false + + CITIZEN: + subclass_of: + - CITY.PERSON + + AGE: + subclass_of: + - CITY.NUMBER + + IMAGE: + subclass_of: + - CUBA.FILE \ No newline at end of file diff --git a/tests/test_yaml2camelcase.py b/tests/test_yaml2camelcase.py new file mode 100644 index 00000000..3cbf56d9 --- /dev/null +++ b/tests/test_yaml2camelcase.py @@ -0,0 +1,34 @@ +import unittest2 as unittest +import yaml +import tempfile +from pathlib import Path + +from osp.core.tools.yaml2camelcase import Yaml2CamelCaseConverter + +caps = Path(__file__).parent / "city_caps.ontology.yml" +camel = Path(__file__).parents[1] / "osp" / "core" / "ontology" / "docs" \ + / "city.ontology.yml" + + +class TestYaml2CamelCase(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + with open(caps) as f: + self.caps_doc = yaml.safe_load(f) + + with open(camel) as f: + self.camel_doc = yaml.safe_load(f) + + def test_convert(self): + converter = Yaml2CamelCaseConverter(caps) + converter.convert() + with tempfile.NamedTemporaryFile("wt") as f: + converter.store(f.name) + + with open(f.name, "rt") as f2: + self.assertEqual(self.camel_doc, yaml.safe_load(f2)) + + +if __name__ == "__main__": + unittest.main() From 736d8848cb4c97e0e743f8b37dadaddc649f29aa Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Mon, 10 Aug 2020 08:38:27 +0200 Subject: [PATCH 02/22] Fix ontology2dot (#461) Fixes #460 --- osp/core/ontology/namespace.py | 7 +++++++ osp/core/ontology/parser.py | 26 ++++++++++++++++++++++++++ osp/core/tools/ontology2dot.py | 28 ++++++++++++++-------------- tests/test_parser.py | 15 +++++++++++++++ 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/osp/core/ontology/namespace.py b/osp/core/ontology/namespace.py index 03644dab..c3f40ba9 100644 --- a/osp/core/ontology/namespace.py +++ b/osp/core/ontology/namespace.py @@ -35,6 +35,13 @@ def __str__(self): def __repr__(self): return "<%s: %s>" % (self._name, self._iri) + def __eq__(self, other): + return self._name == other._name and self._iri == other._iri \ + and self._namespace_registry is other._namespace_registry + + def __hash__(self): + return hash(str(self)) + def get_name(self): """Get the name of the namespace""" return self._name diff --git a/osp/core/ontology/parser.py b/osp/core/ontology/parser.py index 3ed13508..6f7563e7 100644 --- a/osp/core/ontology/parser.py +++ b/osp/core/ontology/parser.py @@ -96,6 +96,32 @@ def get_identifier(file_path_or_doc): else: raise SyntaxError(f"Invalid format of file {file_path_or_doc}") + @staticmethod + def get_namespace_names(file_path_or_doc): + """Get the names of namespaces of the given yaml doc or path to such. + + Args: + file_path_or_doc (Union[Path, dict]): The YAML document or + a path to it + + Raises: + SyntaxError: Invalid YAML format + + Returns: + List[str]: TThe list of defined namespace names. + """ + yaml_doc = file_path_or_doc + if isinstance(file_path_or_doc, str): + file_path = Parser.get_file_path(file_path_or_doc) + with open(file_path, "r") as f: + yaml_doc = yaml.safe_load(f) + if YmlParser.is_yaml_ontology(yaml_doc): + return [YmlParser.get_namespace_name(yaml_doc)] + elif RDF_FILE_KEY in yaml_doc and IDENTIFIER_KEY in yaml_doc: + return [x.lower() for x in yaml_doc[NAMESPACES_KEY].keys()] + else: + raise SyntaxError(f"Invalid format of file {file_path_or_doc}") + @staticmethod def get_requirements(file_path_or_doc): """Get the requirements of a given document or file path to such. diff --git a/osp/core/tools/ontology2dot.py b/osp/core/tools/ontology2dot.py index 6d67d3d1..0e57f2e1 100644 --- a/osp/core/tools/ontology2dot.py +++ b/osp/core/tools/ontology2dot.py @@ -2,7 +2,8 @@ import graphviz import argparse import logging -from osp.core import ONTOLOGY_INSTALLER +from osp.core.namespaces import _namespace_registry +from osp.core.ontology.parser import Parser from osp.core.ontology import OntologyClass, OntologyRelationship, \ OntologyAttribute @@ -31,7 +32,7 @@ def __init__(self, namespaces, output_filename, group=False): self._namespaces = list() for namespace in namespaces: if isinstance(namespace, str): - namespace = ONTOLOGY_INSTALLER.namespace_registry[namespace] + namespace = _namespace_registry[namespace] self._namespaces.append(namespace) self._output_filename = output_filename self._visited = set() @@ -106,7 +107,7 @@ def _add_oclass(self, oclass, graph): :type oclass: OntologyClass """ attr = "" - for key, value in oclass.get_attributes().items(): + for key, value in oclass.attributes.items(): attr += self.attribute.format(key.argname, value) label = self.label.format(str(oclass), attr) if oclass.namespace in self._namespaces: @@ -123,8 +124,6 @@ def _add_relationship(self, rel, graph): :type rel: OntologyRelationship """ attr = "" - for characteristic in rel.characteristics: - attr += self.attribute.format("", characteristic) label = self.label.format(str(rel), attr) if rel.namespace in self._namespaces: graph.node(str(rel), label=label, @@ -187,18 +186,19 @@ def run_from_terminal(): args = parser.parse_args() namespaces = list() - files = list() + parser = Parser(_namespace_registry._graph) for x in args.to_plot: - if x in ONTOLOGY_INSTALLER.namespace_registry: + if x in _namespace_registry: namespaces.append(x) continue - n = ONTOLOGY_INSTALLER._get_namespace(x) - if n in ONTOLOGY_INSTALLER.namespace_registry: - logger.warning("Using installed version of namespace %s" % n) - namespaces.append(ONTOLOGY_INSTALLER.namespace_registry[n]) - else: - files.append(ONTOLOGY_INSTALLER.parser.get_filepath(x)) - namespaces.extend(ONTOLOGY_INSTALLER.parse_files(files)) + for n in Parser.get_namespace_names(x): + if n in _namespace_registry: + logger.warning("Using installed version of namespace %s" % n) + namespaces.append(_namespace_registry[n]) + else: + parser.parse(x) + _namespace_registry.update_namespaces() + namespaces.append(_namespace_registry[n]) # Convert the ontology to dot converter = Ontology2Dot( diff --git a/tests/test_parser.py b/tests/test_parser.py index f3167afe..73788c40 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -141,6 +141,21 @@ def test_get_identifier(self): self.assertEqual(self.parser.get_identifier(YML_FILE), "parser_test") self.assertEqual(self.parser.get_identifier("parser_test"), "parser_test") + self.assertEqual(self.parser.get_identifier("emmo"), "emmo") + + def test_get_namespace_name(self): + self.assertEqual(self.parser.get_namespace_names(YML_DOC), + ["parser_test"]) + self.assertEqual(self.parser.get_namespace_names(YML_FILE), + ["parser_test"]) + self.assertEqual(self.parser.get_namespace_names("parser_test"), + ["parser_test"]) + self.assertEqual(self.parser.get_namespace_names("emmo"), + ['mereotopology', 'physical', 'top', 'semiotics', + 'perceptual', 'reductionistic', 'holistic', + 'physicalistic', 'math', 'properties', 'materials', + 'metrology', 'models', 'manufacturing', 'isq', + 'siunits']) def test_get_requirements(self): self.assertEqual(self.parser.get_requirements(YML_DOC), set()) From 82e7cca7a974c873d191d01ea6144b340833d636 Mon Sep 17 00:00:00 2001 From: aaronAB1993 <39489727+aaronAB1993@users.noreply.github.com> Date: Mon, 10 Aug 2020 16:10:36 +0200 Subject: [PATCH 03/22] fixed postgresql (#462) --- examples/multiple_wrappers_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/multiple_wrappers_example.py b/examples/multiple_wrappers_example.py index c9962e4e..be3a5f51 100644 --- a/examples/multiple_wrappers_example.py +++ b/examples/multiple_wrappers_example.py @@ -19,7 +19,7 @@ db_name = input("Database name: ") host = input("Host: ") port = int(input("Port [5432]: ") or 5432) -postgres_url = 'postgres://%s:%s@%s:%s/%s' % (user, pwd, host, port, db_name) +postgres_url = 'postgresql://%s:%s@%s:%s/%s' % (user, pwd, host, port, db_name) # Let's build an EMMO compatible city! emmo_town = city.City(name='EMMO town') From 56e6658abef64cef236bc3b081e0d8ba649d8b02 Mon Sep 17 00:00:00 2001 From: aaronAB1993 <39489727+aaronAB1993@users.noreply.github.com> Date: Mon, 10 Aug 2020 16:10:56 +0200 Subject: [PATCH 04/22] fixed clear database in case of empty db (#463) --- osp/core/session/db/sql_wrapper_session.py | 32 ++++++++++++---------- tests/test_sqlite_city.py | 10 +++++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/osp/core/session/db/sql_wrapper_session.py b/osp/core/session/db/sql_wrapper_session.py index a1d1eb5d..d410f11b 100644 --- a/osp/core/session/db/sql_wrapper_session.py +++ b/osp/core/session/db/sql_wrapper_session.py @@ -308,20 +308,24 @@ def _clear_database(self): # clear local datastructure from osp.core.namespaces import cuba self._reset_buffers(BufferContext.USER) - self._registry.get(self.root).remove(rel=cuba.relationship) - for uid in list(self._registry.keys()): - if uid != self.root: - del self._registry[uid] - self._reset_buffers(BufferContext.USER) - - # delete the data - for table_name in self._get_table_names( - SqlWrapperSession.CUDS_PREFIX): - self._do_db_delete(table_name, None) - self._do_db_delete(self.RELATIONSHIP_TABLE, None) - self._do_db_delete(self.MASTER_TABLE, None) - self._initialize() - self._commit() + root = self._registry.get(self.root) + + # if there is something to remove + if root.get(rel=cuba.relationship): + root.remove(rel=cuba.relationship) + for uid in list(self._registry.keys()): + if uid != self.root: + del self._registry[uid] + self._reset_buffers(BufferContext.USER) + + # delete the data + for table_name in self._get_table_names( + SqlWrapperSession.CUDS_PREFIX): + self._do_db_delete(table_name, None) + self._do_db_delete(self.RELATIONSHIP_TABLE, None) + self._do_db_delete(self.MASTER_TABLE, None) + self._initialize() + self._commit() except Exception as e: self._rollback_transaction() raise e diff --git a/tests/test_sqlite_city.py b/tests/test_sqlite_city.py index e7550790..2c11e04d 100644 --- a/tests/test_sqlite_city.py +++ b/tests/test_sqlite_city.py @@ -278,6 +278,16 @@ def test_refresh(self): self.assertNotIn(p3w.uid, session._registry) def test_clear_database(self): + # db is empty (no error occurs) + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + session._clear_database() + with SqliteSession(DB) as session: + wrapper = city.CityWrapper(session=session) + wrapper.session.commit() + session._clear_database() + + # db is not empty c = city.City(name="Freiburg") p1 = city.Citizen(name="Peter") p2 = city.Citizen(name="Anna") From abc0617b36052d5a67aebe379f6285e771fc9496 Mon Sep 17 00:00:00 2001 From: aaronAB1993 <39489727+aaronAB1993@users.noreply.github.com> Date: Mon, 10 Aug 2020 16:11:12 +0200 Subject: [PATCH 05/22] empty yaml now throughs meaningful error on pico install (#464) --- osp/core/ontology/parser.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/osp/core/ontology/parser.py b/osp/core/ontology/parser.py index 6f7563e7..3ab8a293 100644 --- a/osp/core/ontology/parser.py +++ b/osp/core/ontology/parser.py @@ -38,6 +38,10 @@ def parse(self, file_path): file_path = self.get_file_path(file_path) with open(file_path, 'r') as f: yaml_doc = yaml.safe_load(f) + if yaml_doc is None: + raise SyntaxError( + f"Empty format of file {file_path}" + ) if YmlParser.is_yaml_ontology(yaml_doc): YmlParser(self.graph).parse(file_path, yaml_doc) elif RDF_FILE_KEY in yaml_doc and IDENTIFIER_KEY in yaml_doc: @@ -89,6 +93,10 @@ def get_identifier(file_path_or_doc): file_path = Parser.get_file_path(file_path_or_doc) with open(file_path, "r") as f: yaml_doc = yaml.safe_load(f) + if yaml_doc is None: + raise SyntaxError( + f"Empty format of file {file_path_or_doc}" + ) if YmlParser.is_yaml_ontology(yaml_doc): return YmlParser.get_namespace_name(yaml_doc) elif RDF_FILE_KEY in yaml_doc and IDENTIFIER_KEY in yaml_doc: From 914d7dffef30c2757fee0442c1726df99d73b190 Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Mon, 10 Aug 2020 16:33:39 +0200 Subject: [PATCH 06/22] Allow to delete CUDS object from a database as a transaction Fixes #421 --- test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test.py diff --git a/test.py b/test.py new file mode 100644 index 00000000..5248984d --- /dev/null +++ b/test.py @@ -0,0 +1,23 @@ +from osp.wrappers.sqlite import SqliteSession +from osp.core.namespaces import city, cuba + + +def delete(db_wrapper, c): + _delete_aux(c, db_wrapper) + db_wrapper.session.commit() + + +def _delete_aux(cuds_object, db_wrapper): + for c in cuds_object.get(rel=cuba.activeRelationship): + _delete_aux(c, db_wrapper) + db_wrapper.session.delete_cuds_object(cuds_object) + # db_wrapper.session.commit() + + +with SqliteSession('test.db') as session: + wrapper = city.CityWrapper(session=session) + a = city.City(name='freiburg', session=wrapper.session) + b = city.Citizen(name='peter', session=wrapper.session) + a.add(b, rel=city.hasInhabitant) + wrapper.session.commit() + delete(wrapper, a) From b71c91c9cd1230bdeff98d23f7bcac0f23b99c47 Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Mon, 10 Aug 2020 16:34:17 +0200 Subject: [PATCH 07/22] uploaded wrong file previously --- osp/core/session/session.py | 2 +- test.py | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 test.py diff --git a/osp/core/session/session.py b/osp/core/session/session.py index 917ab872..ecf8b8ac 100644 --- a/osp/core/session/session.py +++ b/osp/core/session/session.py @@ -76,7 +76,7 @@ def delete_cuds_object(self, cuds_object): from osp.core.namespaces import cuba if cuds_object.session != self: cuds_object = next(self.load(cuds_object.uid)) - if cuds_object.get(): + if cuds_object.get(rel=cuba.relationship): cuds_object.remove(rel=cuba.relationship) del self._registry[cuds_object.uid] self._notify_delete(cuds_object) diff --git a/test.py b/test.py deleted file mode 100644 index 5248984d..00000000 --- a/test.py +++ /dev/null @@ -1,23 +0,0 @@ -from osp.wrappers.sqlite import SqliteSession -from osp.core.namespaces import city, cuba - - -def delete(db_wrapper, c): - _delete_aux(c, db_wrapper) - db_wrapper.session.commit() - - -def _delete_aux(cuds_object, db_wrapper): - for c in cuds_object.get(rel=cuba.activeRelationship): - _delete_aux(c, db_wrapper) - db_wrapper.session.delete_cuds_object(cuds_object) - # db_wrapper.session.commit() - - -with SqliteSession('test.db') as session: - wrapper = city.CityWrapper(session=session) - a = city.City(name='freiburg', session=wrapper.session) - b = city.Citizen(name='peter', session=wrapper.session) - a.add(b, rel=city.hasInhabitant) - wrapper.session.commit() - delete(wrapper, a) From 84d4fbb051d9fea4e9981689de627dbb09c5f7ff Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Mon, 10 Aug 2020 16:39:41 +0200 Subject: [PATCH 08/22] bandit: input is safe in python3 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e76292c1..b12711c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: radon mi -s . - name: bandit - run: bandit -r osp --skip B101 + run: bandit -r osp --skip B101,B322 test: runs-on: self-hosted From 9d2ff5a1cad2e9c75600bc82d150d77488c66a90 Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Tue, 11 Aug 2020 10:04:38 +0200 Subject: [PATCH 09/22] Comments on yaml2camelcase Fixes #467 --- osp/core/tools/yaml2camelcase.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/osp/core/tools/yaml2camelcase.py b/osp/core/tools/yaml2camelcase.py index e3e663ff..ccee9ef0 100644 --- a/osp/core/tools/yaml2camelcase.py +++ b/osp/core/tools/yaml2camelcase.py @@ -35,6 +35,7 @@ def __init__(self, file_path): self.onto_doc = self.doc[ONTOLOGY_KEY] self.orig_onto_doc = deepcopy(self.onto_doc) self.namespace = self.doc[NAMESPACE_KEY].lower() + self.ambiguity_resolution = dict() def convert(self): """Convert the yaml file to CamelCase""" @@ -121,8 +122,12 @@ def get_first_letter_caps(self, word, internal=False): if namespace.lower() == self.namespace: x = self.get_first_letter_caps(name, internal=True) return True if x is None and not internal else x - return input(f"Is {word} a ontology class?") \ + if word in self.ambiguity_resolution: + return self.ambiguity_resolution[word] + ar = input(f"Is {word} a ontology class (y/n)? ") \ .lower().strip().startswith("y") + self.ambiguity_resolution[word] = ar + return ar # unqualified cases if word not in self.orig_onto_doc: From 766d66cad22626351eebbdec7411f7e5c4b7bbd4 Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Tue, 11 Aug 2020 10:11:20 +0200 Subject: [PATCH 10/22] also transform requirements to lower case --- osp/core/tools/yaml2camelcase.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/osp/core/tools/yaml2camelcase.py b/osp/core/tools/yaml2camelcase.py index ccee9ef0..8203d025 100644 --- a/osp/core/tools/yaml2camelcase.py +++ b/osp/core/tools/yaml2camelcase.py @@ -7,7 +7,7 @@ from pathlib import Path from osp.core.ontology.yml.yml_parser import YmlParser from osp.core.ontology.yml.yml_keywords import NAMESPACE_KEY, ONTOLOGY_KEY, \ - SUPERCLASSES_KEY + SUPERCLASSES_KEY, REQUIREMENTS_KEY entity_name_regex = r"(_|[A-Z])([A-Z]|[0-9]|_)*" @@ -40,6 +40,8 @@ def __init__(self, file_path): def convert(self): """Convert the yaml file to CamelCase""" self.doc[NAMESPACE_KEY] = self.namespace + self.doc[REQUIREMENTS_KEY] = [x.lower() + for x in self.doc[REQUIREMENTS_KEY]] self.convert_nested_doc(self.onto_doc, pattern=entity_name_pattern) def convert_nested_doc(self, doc, pattern=qualified_entity_name_pattern): From 3aaa96644615db2f75699d9aebfab1a74bb86934 Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Tue, 11 Aug 2020 10:13:49 +0200 Subject: [PATCH 11/22] fixed unittests --- osp/core/tools/yaml2camelcase.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/osp/core/tools/yaml2camelcase.py b/osp/core/tools/yaml2camelcase.py index 8203d025..a396023e 100644 --- a/osp/core/tools/yaml2camelcase.py +++ b/osp/core/tools/yaml2camelcase.py @@ -40,8 +40,9 @@ def __init__(self, file_path): def convert(self): """Convert the yaml file to CamelCase""" self.doc[NAMESPACE_KEY] = self.namespace - self.doc[REQUIREMENTS_KEY] = [x.lower() - for x in self.doc[REQUIREMENTS_KEY]] + if REQUIREMENTS_KEY in self.doc: + self.doc[REQUIREMENTS_KEY] = [x.lower() + for x in self.doc[REQUIREMENTS_KEY]] self.convert_nested_doc(self.onto_doc, pattern=entity_name_pattern) def convert_nested_doc(self, doc, pattern=qualified_entity_name_pattern): From cb8e109a1afdd6336161b9969258fa1cb02a1da8 Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Tue, 11 Aug 2020 10:16:38 +0200 Subject: [PATCH 12/22] flake8 --- osp/core/tools/yaml2camelcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osp/core/tools/yaml2camelcase.py b/osp/core/tools/yaml2camelcase.py index a396023e..04b881ea 100644 --- a/osp/core/tools/yaml2camelcase.py +++ b/osp/core/tools/yaml2camelcase.py @@ -42,7 +42,7 @@ def convert(self): self.doc[NAMESPACE_KEY] = self.namespace if REQUIREMENTS_KEY in self.doc: self.doc[REQUIREMENTS_KEY] = [x.lower() - for x in self.doc[REQUIREMENTS_KEY]] + for x in self.doc[REQUIREMENTS_KEY]] self.convert_nested_doc(self.onto_doc, pattern=entity_name_pattern) def convert_nested_doc(self, doc, pattern=qualified_entity_name_pattern): From 80108afcbe655b5b4b403acad200b37eb219824c Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Tue, 11 Aug 2020 10:41:57 +0200 Subject: [PATCH 13/22] avoid overwriting backup file in yaml2camelcase --- osp/core/tools/yaml2camelcase.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/osp/core/tools/yaml2camelcase.py b/osp/core/tools/yaml2camelcase.py index 04b881ea..097c98f9 100644 --- a/osp/core/tools/yaml2camelcase.py +++ b/osp/core/tools/yaml2camelcase.py @@ -3,6 +3,7 @@ import logging import re import shutil +import os from copy import deepcopy from pathlib import Path from osp.core.ontology.yml.yml_parser import YmlParser @@ -88,7 +89,12 @@ def store(self, output): output = self.file_path if output == self.file_path: logger.info(f"Backing up original file at {output}.orig") - shutil.copyfile(str(output), str(output) + ".orig") + orig_path = f"{output}.orig" + orig_counter = 0 + while os.path.exists(orig_path): + orig_path = f"{output}.orig[{orig_counter}]" + orig_counter += 1 + shutil.copyfile(str(output), orig_path) logger.info(f"Writing camel case file to {output}") with open(output, "w") as f: From b8a208393280ccafd6f5e4a088f3d9d169104957 Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Tue, 11 Aug 2020 10:52:44 +0200 Subject: [PATCH 14/22] Add easy way to serialize CUDS objects (#468) * Add easy way to serialize CUDS objects Fixes #322 * Fixed unittest --- osp/core/utils/general.py | 46 +++++++++++++++++++++++++++++++++------ tests/test_utils.py | 43 +++++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/osp/core/utils/general.py b/osp/core/utils/general.py index a694758b..1069f2be 100644 --- a/osp/core/utils/general.py +++ b/osp/core/utils/general.py @@ -2,6 +2,7 @@ import json import rdflib from osp.core.namespaces import cuba +from osp.core.session.buffers import BufferContext def branch(cuds_object, *args, rel=None): @@ -74,7 +75,36 @@ def post(url, cuds_object, max_depth=float("inf")): headers={"content_type": "application/json"}) -def deserialize(json_doc, session=None): +def serialize(cuds_object, rel=cuba.activeRelationship, + max_depth=float("inf"), json_dumps=True): + """Serialize a cuds objects and all of its contents recursively. + + Args: + cuds_object (Cuds): The cuds object to serialize + rel (OntologyRelationship, optional): The relationships to follow when + serializing recursively. Defaults to cuba.activeRelationship. + max_depth (int, optional): The maximum recursion depth. + Defaults to float("inf"). + json_dumps (bool, optional): Whether to dump it to the registry. + Defaults to True. + + Returns: + Union[str, List]: The serialized cuds object. + """ + from osp.core.session.transport.transport_utils import serializable + from osp.core.utils import find_cuds_object + cuds_objects = find_cuds_object(criterion=lambda x: True, + root=cuds_object, + rel=rel, + find_all=True, + max_depth=max_depth) + result = serializable(cuds_objects) + if json_dumps: + return json.dumps(result) + return result + + +def deserialize(json_doc, session=None, buffer_context=BufferContext.USER): """Deserialize the given json objects (to CUDS). Will add the CUDS objects to the buffers. @@ -83,21 +113,23 @@ def deserialize(json_doc, session=None): Either string or already loaded json object. session (Session, optional): The session to add the CUDS objects to. Defaults to the CoreSession. + buffer_context (BufferContext): Whether to add the objects to the + buffers of the user or the engine. Default is equivalent of + the user creating the CUDS objects by hand. Returns: Any: The deserialized data. Can be CUDS. """ from osp.core.cuds import Cuds - from osp.core.session.buffers import BufferContext - from osp.core.session.transport.transport_utils import deserialize as x - + from osp.core.session.transport.transport_utils import deserialize \ + as _deserialize if isinstance(json_doc, str): json_doc = json.loads(json_doc) session = session or Cuds._session - return x( + return _deserialize( json_obj=json_doc, - session=session, # The core session - buffer_context=BufferContext.USER + session=session, + buffer_context=buffer_context ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 020e3db3..246eb768 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,6 +4,7 @@ import uuid import os import osp.core +import json from osp.core.namespaces import cuba from osp.core.session.transport.transport_utils import serializable from osp.core.session.core_session import CoreSession @@ -19,7 +20,7 @@ find_cuds_objects_by_attribute, post, get_relationships_between, get_neighbor_diff, change_oclass, branch, validate_tree_against_schema, - ConsistencyError, CardinalityError + ConsistencyError, CardinalityError, serialize ) from osp.core.cuds import Cuds @@ -46,6 +47,21 @@ } } +CUDS_LIST = [ + {"oclass": "city.City", "uid": str(uuid.UUID(int=1)), + "attributes": {"coordinates": [0, 0], "name": "Freiburg"}, + "relationships": { + "city.hasPart": {str(uuid.UUID(int=2)): "city.Neighborhood"}}}, + {"oclass": "city.Neighborhood", "uid": str(uuid.UUID(int=2)), + "attributes": {"coordinates": [0, 0], "name": "Littenweiler"}, + "relationships": {"city.hasPart": {str(uuid.UUID(int=3)): "city.Street"}, + "city.isPartOf": {str(uuid.UUID(int=1)): "city.City"}}}, + {"oclass": "city.Street", "uid": str(uuid.UUID(int=3)), + "attributes": {"coordinates": [0, 0], "name": "Schwarzwaldstraße"}, + "relationships": { + "city.isPartOf": {str(uuid.UUID(int=2)): "city.Neighborhood"}}} +] + def get_test_city(): """helper function""" @@ -201,6 +217,31 @@ def test_deserialize(self): self.assertTrue(result.is_a(city.Citizen)) self.assertEqual(result.name, "Peter") self.assertEqual(result.age, 23) + result = deserialize([CUDS_DICT]) + self.assertEqual(len(result), 1) + self.assertTrue(result[0].is_a(city.Citizen)) + self.assertEqual(result[0].name, "Peter") + self.assertEqual(result[0].age, 23) + self.assertEqual(CUDS_LIST, serialize(deserialize(CUDS_LIST)[0], + json_dumps=False)) + + def test_serialize(self): + c = branch( + city.City(name="Freiburg", uid=1), + branch( + city.Neighborhood(name="Littenweiler", uid=2), + city.Street(name="Schwarzwaldstraße", uid=3) + ) + ) + self.maxDiff = None + self.assertEqual( + json.loads(serialize(c)), + CUDS_LIST + ) + self.assertEqual( + serialize(c, json_dumps=False), + CUDS_LIST + ) def test_clone_cuds_object(self): """Test cloning of cuds""" From 5569393c05949d65bc9247ad1f559f5f358c0321 Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Tue, 11 Aug 2020 11:00:55 +0200 Subject: [PATCH 15/22] added method to delete cuds objects recursively (#466) --- osp/core/utils/general.py | 20 ++++++++++++++++++++ tests/test_utils.py | 24 +++++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/osp/core/utils/general.py b/osp/core/utils/general.py index 1069f2be..9127e2bb 100644 --- a/osp/core/utils/general.py +++ b/osp/core/utils/general.py @@ -24,6 +24,26 @@ def branch(cuds_object, *args, rel=None): return cuds_object +def delete_cuds_object_recursively(cuds_object, rel=cuba.activeRelationship, + max_depth=float("inf")): + """Delete a cuds object and all the object inside of the container + of it. + + Args: + cuds_object (Cuds): The Cuds object to recursively delete + max_depth (int, optional): The maximum depth of the recursion. + Defaults to float("inf"). + """ + from osp.core.utils.simple_search import find_cuds_object + cuds_objects = find_cuds_object(criterion=lambda x: True, + root=cuds_object, + rel=rel, + find_all=True, + max_depth=max_depth) + for obj in cuds_objects: + obj.session.delete_cuds_object(obj) + + def get_rdf_graph(session=None): """EXPERIMENTAL Get the RDF Graph from a session. diff --git a/tests/test_utils.py b/tests/test_utils.py index 246eb768..1c0eefdc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,8 +20,10 @@ find_cuds_objects_by_attribute, post, get_relationships_between, get_neighbor_diff, change_oclass, branch, validate_tree_against_schema, - ConsistencyError, CardinalityError, serialize + ConsistencyError, CardinalityError, delete_cuds_object_recursively, + serialize ) +from osp.core.session.buffers import BufferContext from osp.core.cuds import Cuds try: @@ -645,3 +647,23 @@ def test_pretty_print(self): " uuid: %s" % s1.uid, " (already printed)", ""])) + + def test_delete_cuds_object_recursively(self): + with TestWrapperSession() as session: + wrapper = city.CityWrapper(session=session) + a = city.City(name='freiburg', session=session) + b = city.Citizen(name='peter', session=session) + branch( + wrapper, + branch(a, b, rel=city.hasInhabitant) + ) + self.maxDiff = None + session._reset_buffers(BufferContext.USER) + delete_cuds_object_recursively(a) + self.assertEqual(session._buffers, [ + [{}, {wrapper.uid: wrapper}, {a.uid: a, b.uid: b}], + [{}, {}, {}], + ]) + self.assertEqual(wrapper.get(rel=cuba.relationship), []) + self.assertEqual(a.get(rel=cuba.relationship), []) + self.assertEqual(b.get(rel=cuba.relationship), []) From ae74fb116a015e31c2b189e012728d00f7d2a306 Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Tue, 11 Aug 2020 11:22:26 +0200 Subject: [PATCH 16/22] Communicate that yaml2camelcase exists (#473) * Communicate that yaml2camelcase exists Fixes #472 * update packageinfo --- osp/core/cuds.py | 6 ++++-- osp/core/ontology/namespace.py | 4 +++- osp/core/ontology/oclass.py | 3 ++- osp/core/ontology/yml/yml_parser.py | 4 +++- packageinfo.py | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/osp/core/cuds.py b/osp/core/cuds.py index ae82aa49..ce6c7c6e 100644 --- a/osp/core/cuds.py +++ b/osp/core/cuds.py @@ -806,7 +806,8 @@ def __getattr__(self, name): f"Note that you must match the case of the definition in " f"the ontology in future releases. Additionally, entity " f"names defined in YAML ontology are no longer required " - f"to be ALL_CAPS." + f"to be ALL_CAPS. You can use the yaml2camelcase " + f"commandline tool to transform entity names to CamelCase." ) return self._attr_values[name.upper()] else: @@ -840,7 +841,8 @@ def __setattr__(self, name, new_value): f"Note that you must match the case of the definition in " f"the ontology in future releases. Additionally, entity " f"names defined in YAML ontology are no longer required " - f"to be ALL_CAPS." + f"to be ALL_CAPS. You can use the yaml2camelcase " + f"commandline tool to transform entity names to CamelCase." ) return self._attr_values[name.upper()] raise AttributeError(name) diff --git a/osp/core/ontology/namespace.py b/osp/core/ontology/namespace.py index c3f40ba9..be4cf08b 100644 --- a/osp/core/ontology/namespace.py +++ b/osp/core/ontology/namespace.py @@ -193,7 +193,9 @@ def _get_case_insensitive(self, name): f"{alternative} is referenced with '{name}'. " f"Note that referencing entities will be case sensitive " f"in future releases. Additionally, entity names defined " - f"in YAML ontology are no longer required to be ALL_CAPS." + f"in YAML ontology are no longer required to be ALL_CAPS. " + f"You can use the yaml2camelcase " + f"commandline tool to transform entity names to CamelCase." ) return r except KeyError as e: diff --git a/osp/core/ontology/oclass.py b/osp/core/ontology/oclass.py index 97393c91..3f105a44 100644 --- a/osp/core/ontology/oclass.py +++ b/osp/core/ontology/oclass.py @@ -117,7 +117,8 @@ def _get_attributes_values(self, kwargs, _force): f"Note that you must match the case of the definition in " f"the ontology in future releases. Additionally, entity " f"names defined in YAML ontology are no longer required " - f"to be ALL_CAPS." + f"to be ALL_CAPS. You can use the yaml2camelcase " + f"commandline tool to transform entity names to CamelCase." ) else: attributes[attribute] = default diff --git a/osp/core/ontology/yml/yml_parser.py b/osp/core/ontology/yml/yml_parser.py index 83bc6681..b3785ec8 100644 --- a/osp/core/ontology/yml/yml_parser.py +++ b/osp/core/ontology/yml/yml_parser.py @@ -214,7 +214,9 @@ def _get_iri_case_insensitive(self, entity_name, namespace, f"{current_entity}. " f"Note that referencing entities will be case sensitive " f"in future releases. Additionally, entity names defined " - f"in YAML ontology are no longer required to be ALL_CAPS." + f"in YAML ontology are no longer required to be ALL_CAPS. " + f"You can use the yaml2camelcase " + f"commandline tool to transform entity names to CamelCase." ) return r except AttributeError as e: diff --git a/packageinfo.py b/packageinfo.py index a93d09e4..b51b13b0 100644 --- a/packageinfo.py +++ b/packageinfo.py @@ -1,2 +1,2 @@ NAME = "osp-core" -VERSION = "3.4.0" +VERSION = "3.4.1" From 285638ade04422817d6cfed754231e8bd5babe5f Mon Sep 17 00:00:00 2001 From: Matthias Urban Date: Tue, 11 Aug 2020 14:27:34 +0200 Subject: [PATCH 17/22] Error importing validate_tree_against_schema Fixes #475 --- osp/core/utils/general.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/osp/core/utils/general.py b/osp/core/utils/general.py index 9127e2bb..833bb4a4 100644 --- a/osp/core/utils/general.py +++ b/osp/core/utils/general.py @@ -2,7 +2,6 @@ import json import rdflib from osp.core.namespaces import cuba -from osp.core.session.buffers import BufferContext def branch(cuds_object, *args, rel=None): @@ -124,7 +123,7 @@ def serialize(cuds_object, rel=cuba.activeRelationship, return result -def deserialize(json_doc, session=None, buffer_context=BufferContext.USER): +def deserialize(json_doc, session=None, buffer_context=None): """Deserialize the given json objects (to CUDS). Will add the CUDS objects to the buffers. @@ -143,9 +142,11 @@ def deserialize(json_doc, session=None, buffer_context=BufferContext.USER): from osp.core.cuds import Cuds from osp.core.session.transport.transport_utils import deserialize \ as _deserialize + from osp.core.session.buffers import BufferContext if isinstance(json_doc, str): json_doc = json.loads(json_doc) session = session or Cuds._session + buffer_context = buffer_context or BufferContext.USER return _deserialize( json_obj=json_doc, session=session, From 06483918a1d594c7bf65f774bf099cecb55d0dd0 Mon Sep 17 00:00:00 2001 From: aaronAB1993 <39489727+aaronAB1993@users.noreply.github.com> Date: Wed, 12 Aug 2020 09:22:01 +0200 Subject: [PATCH 18/22] removed license text (#479) --- osp/core/ontology/attribute.py | 7 ------- osp/core/ontology/entity.py | 7 ------- osp/core/ontology/namespace.py | 7 ------- osp/core/ontology/namespace_registry.py | 7 ------- osp/core/ontology/oclass.py | 8 -------- osp/core/ontology/relationship.py | 7 ------- tests/sqlite_performance_test.py | 7 ------- 7 files changed, 50 deletions(-) diff --git a/osp/core/ontology/attribute.py b/osp/core/ontology/attribute.py index 07869f47..37f097ac 100644 --- a/osp/core/ontology/attribute.py +++ b/osp/core/ontology/attribute.py @@ -1,10 +1,3 @@ -# Copyright (c) 2018, Adham Hashibon and Materials Informatics Team -# at Fraunhofer IWM. -# All rights reserved. -# Redistribution and use are limited to the scope agreed with the end user. -# No parts of this software may be used outside of this context. -# No redistribution is allowed without explicit written permission. - from osp.core.ontology.entity import OntologyEntity from osp.core.ontology.datatypes import convert_from, convert_to import logging diff --git a/osp/core/ontology/entity.py b/osp/core/ontology/entity.py index 8575b347..29ab7285 100644 --- a/osp/core/ontology/entity.py +++ b/osp/core/ontology/entity.py @@ -1,10 +1,3 @@ -# Copyright (c) 2018, Adham Hashibon and Materials Informatics Team -# at Fraunhofer IWM. -# All rights reserved. -# Redistribution and use are limited to the scope agreed with the end user. -# No parts of this software may be used outside of this context. -# No redistribution is allowed without explicit written permission. - from abc import ABC, abstractmethod import rdflib import logging diff --git a/osp/core/ontology/namespace.py b/osp/core/ontology/namespace.py index be4cf08b..f8ca1850 100644 --- a/osp/core/ontology/namespace.py +++ b/osp/core/ontology/namespace.py @@ -1,10 +1,3 @@ -# Copyright (c) 2018, Adham Hashibon and Materials Informatics Team -# at Fraunhofer IWM. -# All rights reserved. -# Redistribution and use are limited to the scope agreed with the end user. -# No parts of this software may be used outside of this context. -# No redistribution is allowed without explicit written permission. - import rdflib import logging diff --git a/osp/core/ontology/namespace_registry.py b/osp/core/ontology/namespace_registry.py index 832e8e27..2876f6ef 100644 --- a/osp/core/ontology/namespace_registry.py +++ b/osp/core/ontology/namespace_registry.py @@ -1,10 +1,3 @@ -# Copyright (c) 2014-2019, Adham Hashibon, Materials Informatics Team, -# Fraunhofer IWM. -# All rights reserved. -# Redistribution and use are limited to the scope agreed with the end user. -# No parts of this software may be used outside of this context. -# No redistribution is allowed without explicit written permission. - import os import logging import rdflib diff --git a/osp/core/ontology/oclass.py b/osp/core/ontology/oclass.py index 3f105a44..f70ba43d 100644 --- a/osp/core/ontology/oclass.py +++ b/osp/core/ontology/oclass.py @@ -1,11 +1,3 @@ -# Copyright (c) 2018, Adham Hashibon and Materials Informatics Team -# at Fraunhofer IWM. -# All rights reserved. -# Redistribution and use are limited to the scope agreed with the end user. -# No parts of this software may be used outside of this context. -# No redistribution is allowed without explicit written permission. - - from osp.core.ontology.entity import OntologyEntity from osp.core.ontology.cuba import rdflib_cuba import logging diff --git a/osp/core/ontology/relationship.py b/osp/core/ontology/relationship.py index 189fbedc..ac0fc138 100644 --- a/osp/core/ontology/relationship.py +++ b/osp/core/ontology/relationship.py @@ -1,10 +1,3 @@ -# Copyright (c) 2018, Adham Hashibon and Materials Informatics Team -# at Fraunhofer IWM. -# All rights reserved. -# Redistribution and use are limited to the scope agreed with the end user. -# No parts of this software may be used outside of this context. -# No redistribution is allowed without explicit written permission. - from osp.core.ontology.entity import OntologyEntity import logging import rdflib diff --git a/tests/sqlite_performance_test.py b/tests/sqlite_performance_test.py index 6361f81d..13daa9f9 100644 --- a/tests/sqlite_performance_test.py +++ b/tests/sqlite_performance_test.py @@ -1,10 +1,3 @@ -# Copyright (c) 2018, Adham Hashibon and Materials Informatics Team -# at Fraunhofer IWM. -# All rights reserved. -# Redistribution and use are limited to the scope agreed with the end user. -# No parts of this software may be used outside of this context. -# No redistribution is allowed without explicit written permission. - # pip install pympler import gc import os From 0b8695220d795a530d81f32108fb1f97ac01b28f Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Wed, 12 Aug 2020 09:24:50 +0200 Subject: [PATCH 19/22] Fix spelling mistake in disambiguation question of yaml2camelcase Fixes #480 (#481) --- examples/ontology_example.py | 2 +- osp/core/tools/yaml2camelcase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/ontology_example.py b/examples/ontology_example.py index ffe075dc..40baca6e 100644 --- a/examples/ontology_example.py +++ b/examples/ontology_example.py @@ -92,7 +92,7 @@ print(city.Citizen(name="Test Person", age=42)) print("Take a look at api_example.py for a description of the CUDS API") -print("\nYou can check if a CUDS object is an instace of a ontology class") +print("\nYou can check if a CUDS object is an instace of an ontology class") print(city.Citizen(name="Test Person", age=42).is_a(city.Citizen)) print(city.Citizen(name="Test Person", age=42).is_a(city.LivingBeing)) diff --git a/osp/core/tools/yaml2camelcase.py b/osp/core/tools/yaml2camelcase.py index 097c98f9..b13611b8 100644 --- a/osp/core/tools/yaml2camelcase.py +++ b/osp/core/tools/yaml2camelcase.py @@ -133,7 +133,7 @@ def get_first_letter_caps(self, word, internal=False): return True if x is None and not internal else x if word in self.ambiguity_resolution: return self.ambiguity_resolution[word] - ar = input(f"Is {word} a ontology class (y/n)? ") \ + ar = input(f"Is {word} an ontology class (y/n)? ") \ .lower().strip().startswith("y") self.ambiguity_resolution[word] = ar return ar From 16d8ea416f19b2b62fba72401b98ef6b56525738 Mon Sep 17 00:00:00 2001 From: aaronAB1993 <39489727+aaronAB1993@users.noreply.github.com> Date: Wed, 12 Aug 2020 09:59:43 +0200 Subject: [PATCH 20/22] fixed get rdfgrpah (#477) * fixed get rdfgrpah * minor * added test to check union Co-authored-by: Matthias Urban --- osp/core/cuds.py | 3 +-- osp/core/utils/general.py | 19 +++++++++++-------- tests/test_utils.py | 26 +++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/osp/core/cuds.py b/osp/core/cuds.py index ce6c7c6e..4357f783 100644 --- a/osp/core/cuds.py +++ b/osp/core/cuds.py @@ -764,8 +764,7 @@ def _iri_from_uid(self, uid): Returns: URIRef: The IRI of the CUDS object with the given UUID. """ - from osp.core import IRI_DOMAIN - return rdflib.URIRef(IRI_DOMAIN + "/#%s" % uid) + return rdflib.URIRef("http://www.osp-core.com/cuds/#%s" % uid) def __str__(self) -> str: """ diff --git a/osp/core/utils/general.py b/osp/core/utils/general.py index 833bb4a4..82f538ca 100644 --- a/osp/core/utils/general.py +++ b/osp/core/utils/general.py @@ -2,6 +2,7 @@ import json import rdflib from osp.core.namespaces import cuba +from osp.core.session.session import Session def branch(cuds_object, *args, rel=None): @@ -55,17 +56,19 @@ def get_rdf_graph(session=None): Returns: rdflib.Graph: The resulting rdf Graph """ + if session is not None: + if not isinstance(session, Session): + raise TypeError( + f"Invalid argument: {session}." + f"Function can only be called on (sub)classes of {Session}.""" + ) from osp.core.cuds import Cuds - from osp.core import ONTOLOGY_NAMESPACE_REGISTRY + from osp.core.namespaces import _namespace_registry session = session or Cuds._session - graph = rdflib.Graph() + cuds_graph = rdflib.Graph() for triple in session.get_triples(): - graph.add(triple) - for namespace in ONTOLOGY_NAMESPACE_REGISTRY: - for entity in namespace: - for triple in entity.get_triples(): - graph.add(triple) - return graph + cuds_graph.add(triple) + return cuds_graph + _namespace_registry._graph def post(url, cuds_object, max_depth=float("inf")): diff --git a/tests/test_utils.py b/tests/test_utils.py index 1c0eefdc..378d6026 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,6 +4,7 @@ import uuid import os import osp.core +import rdflib import json from osp.core.namespaces import cuba from osp.core.session.transport.transport_utils import serializable @@ -20,7 +21,8 @@ find_cuds_objects_by_attribute, post, get_relationships_between, get_neighbor_diff, change_oclass, branch, validate_tree_against_schema, - ConsistencyError, CardinalityError, delete_cuds_object_recursively, + ConsistencyError, CardinalityError, get_rdf_graph, + delete_cuds_object_recursively, serialize ) from osp.core.session.buffers import BufferContext @@ -87,6 +89,28 @@ def get_test_city(): class TestUtils(unittest.TestCase): + def test_get_rdf_graph(self): + with TestWrapperSession() as session: + wrapper = cuba.Wrapper(session=session) + c = city.City(name='freiburg', session=session) + wrapper.add(c, rel=cuba.activeRelationship) + graph = get_rdf_graph(c.session) + + # cuds must be in the grap + iri = rdflib.URIRef( + "http://www.osp-core.com/cuds/#%s" % c.uid + ) + subjects = list(graph.subjects()) + self.assertTrue(iri in subjects) + # ontology entities must be in the graph + cuba_entity_iri = rdflib.URIRef( + "http://www.osp-core.com/cuba#Entity" + ) + self.assertTrue(cuba_entity_iri in subjects) + # fail on invalid arguments + self.assertRaises(TypeError, get_rdf_graph, c) + self.assertRaises(TypeError, get_rdf_graph, 42) + def test_validate_tree_against_schema(self): """Test validation of CUDS tree against schema.yml""" schema_file = os.path.join( From b09f466d555b7c7da60541d21b9e8f8d7ebd45bd Mon Sep 17 00:00:00 2001 From: aaronAB1993 <39489727+aaronAB1993@users.noreply.github.com> Date: Thu, 13 Aug 2020 08:15:40 +0200 Subject: [PATCH 21/22] default relationship can now refer to different namespace (#478) --- osp/core/ontology/yml/yml_parser.py | 54 +++++++++++++++++-- osp/core/ontology/yml/yml_validator.py | 1 + ...t_rel_across_namespace_two_definitions.yml | 13 +++++ ...el_across_namespace_uninstalled_entity.yml | 12 +++++ ...across_namespace_uninstalled_namespace.yml | 12 +++++ tests/default_rel_across_namespace_valid.yml | 12 +++++ tests/test_namespace.py | 28 ++++++++++ tests/test_yml_parser.py | 14 +++-- 8 files changed, 138 insertions(+), 8 deletions(-) create mode 100644 tests/default_rel_across_namespace_two_definitions.yml create mode 100644 tests/default_rel_across_namespace_uninstalled_entity.yml create mode 100644 tests/default_rel_across_namespace_uninstalled_namespace.yml create mode 100644 tests/default_rel_across_namespace_valid.yml diff --git a/osp/core/ontology/yml/yml_parser.py b/osp/core/ontology/yml/yml_parser.py index b3785ec8..c686cddf 100644 --- a/osp/core/ontology/yml/yml_parser.py +++ b/osp/core/ontology/yml/yml_parser.py @@ -87,12 +87,14 @@ def _parse_ontology(self): t = self._validate_entity(entity_name, entity_doc) types[entity_name] = t + self._assert_default_relationship_occurrence() + self._check_default_rel_definition_on_ontology() for entity_name, entity_doc in self._ontology_doc.items(): # self._load_class_expressions(entity) TODO if types[entity_name] == "relationship": self._set_inverse(entity_name, entity_doc) # self._parse_rel_characteristics(entity_name, entity_doc) TODO - self._check_default_rel(entity_name, entity_doc) + self._check_default_rel_flag_on_entity(entity_name, entity_doc) elif types[entity_name] == "attribute": self._set_datatype(entity_name, entity_doc) @@ -404,8 +406,54 @@ def _set_inverse(self, entity_name, entity_doc): (iri, rdflib.OWL.inverseOf, inverse_iri) ) - def _check_default_rel(self, entity_name, entity_doc): - """Check if the given relationship the default + def _assert_default_relationship_occurrence(self): + """Assures that only one default relationship is defined in the yaml + + :raises ValueError: If more than one definition is found. + """ + occurrences = 0 + if DEFAULT_REL_KEY in self._doc: + occurrences += 1 + for entity_name, entity_doc in self._ontology_doc.items(): + if DEFAULT_REL_KEY in entity_doc: + occurrences += 1 + if occurrences > 1: + raise ValueError( + f"You have defined {occurrences} default relationships for " + f"namespace {self._namespace} although <= 1 are allowed." + ) + + def _check_default_rel_definition_on_ontology(self): + """Check if the given yaml defines + a default relationship, save that accordingly. + """ + if DEFAULT_REL_KEY in self._doc: + namespace, entity_name = self._doc[DEFAULT_REL_KEY].split('.') + + # defined relationship must be installed + from osp.core.namespaces import _namespace_registry + referred_namespace = _namespace_registry.get(namespace) + if not referred_namespace: + raise ValueError( + f"The namespace {namespace} that you have defined for " + f"the default relationship \"{entity_name}\" of " + f"namespace \"{self._namespace}\" is not installed." + ) + referred_entity = referred_namespace.get(entity_name) + if not referred_entity: + raise ValueError( + f"The default relationship \"{entity_name}\" from " + f"\"{namespace}\" that you have defined for namespace " + f"\"{self._namespace}\" is not installed." + ) + + self.graph.add( + (self._get_iri(), rdflib_cuba._default_rel, + self._get_iri(namespace=namespace, entity_name=entity_name)) + ) + + def _check_default_rel_flag_on_entity(self, entity_name, entity_doc): + """Check if the given relationship is the default When it is a default, save that accordingly. Args: diff --git a/osp/core/ontology/yml/yml_validator.py b/osp/core/ontology/yml/yml_validator.py index 4fe60374..c84808aa 100644 --- a/osp/core/ontology/yml/yml_validator.py +++ b/osp/core/ontology/yml/yml_validator.py @@ -48,6 +48,7 @@ "!" + VERSION_KEY: re.compile(r"^\d+\.\d+(\.\d+)?$"), "!" + NAMESPACE_KEY: namespace_name_pattern, "!" + ONTOLOGY_KEY: {entity_name_pattern: "entity_def"}, + DEFAULT_REL_KEY: qualified_entity_name_pattern, AUTHOR_KEY: str, REQUIREMENTS_KEY: [entity_name_pattern] }, diff --git a/tests/default_rel_across_namespace_two_definitions.yml b/tests/default_rel_across_namespace_two_definitions.yml new file mode 100644 index 00000000..3056450f --- /dev/null +++ b/tests/default_rel_across_namespace_two_definitions.yml @@ -0,0 +1,13 @@ +--- + version: "0.0.3" + + namespace: "default_rel_test_namespace_two_definitions" + default_rel: cuba.activeRelationship + ontology: + SomeEntity: + subclass_of: + - cuba.Entity + someRel: + subclass_of: + - cuba.activeRelationship + default_rel: true \ No newline at end of file diff --git a/tests/default_rel_across_namespace_uninstalled_entity.yml b/tests/default_rel_across_namespace_uninstalled_entity.yml new file mode 100644 index 00000000..94134192 --- /dev/null +++ b/tests/default_rel_across_namespace_uninstalled_entity.yml @@ -0,0 +1,12 @@ +--- + version: "0.0.3" + + namespace: "default_rel_test_namespace_uninstalled_entity" + default_rel: cuba.noneExistingRel + ontology: + SomeEntity: + subclass_of: + - cuba.Entity + someRel: + subclass_of: + - cuba.activeRelationship \ No newline at end of file diff --git a/tests/default_rel_across_namespace_uninstalled_namespace.yml b/tests/default_rel_across_namespace_uninstalled_namespace.yml new file mode 100644 index 00000000..da91efc5 --- /dev/null +++ b/tests/default_rel_across_namespace_uninstalled_namespace.yml @@ -0,0 +1,12 @@ +--- + version: "0.0.3" + + namespace: "default_rel_test_namespace_uninstalled_namespace" + default_rel: non_existing_namespace.activeRelationship + ontology: + SomeEntity: + subclass_of: + - cuba.Entity + someRel: + subclass_of: + - cuba.activeRelationship \ No newline at end of file diff --git a/tests/default_rel_across_namespace_valid.yml b/tests/default_rel_across_namespace_valid.yml new file mode 100644 index 00000000..3f148e86 --- /dev/null +++ b/tests/default_rel_across_namespace_valid.yml @@ -0,0 +1,12 @@ +--- + version: "0.0.3" + + namespace: "default_rel_test_namespace_valid" + default_rel: cuba.activeRelationship + ontology: + SomeEntity: + subclass_of: + - cuba.Entity + someRel: + subclass_of: + - cuba.activeRelationship \ No newline at end of file diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 87b4c92c..df39fa6f 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -8,6 +8,7 @@ from osp.core.ontology.installation import OntologyInstallationManager from osp.core.ontology import OntologyClass, OntologyRelationship, \ OntologyAttribute +from osp.core.namespaces import cuba CUBA_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "osp", "core", "ontology", "docs", "cuba.ttl") @@ -312,10 +313,37 @@ def test_namespace_str(self): "") def test_get_default_rel(self): + # default rel defined as flag in entity name self.installer.install("city") namespace = self.namespace_registry.city self.assertEqual(namespace.get_default_rel().name, "hasPart") + onto_def_rel = os.path.join( + os.path.dirname(__file__), + 'default_rel_across_namespace_valid.yml' + ) + self.installer.install(onto_def_rel) + namespace = self.namespace_registry.default_rel_test_namespace_valid + self.assertEqual(namespace.get_default_rel(), cuba.activeRelationship) + + onto_def_rel = os.path.join( + os.path.dirname(__file__), + 'default_rel_across_namespace_two_definitions.yml' + ) + self.assertRaises(ValueError, self.installer.install, onto_def_rel) + + onto_def_rel = os.path.join( + os.path.dirname(__file__), + 'default_rel_across_namespace_uninstalled_entity.yml' + ) + self.assertRaises(ValueError, self.installer.install, onto_def_rel) + + onto_def_rel = os.path.join( + os.path.dirname(__file__), + 'default_rel_across_namespace_uninstalled_namespace.yml' + ) + self.assertRaises(ValueError, self.installer.install, onto_def_rel) + def test_contains(self): self.installer.install("city") namespace = self.namespace_registry.city diff --git a/tests/test_yml_parser.py b/tests/test_yml_parser.py index 36323243..a3450433 100644 --- a/tests/test_yml_parser.py +++ b/tests/test_yml_parser.py @@ -73,12 +73,16 @@ def test_set_datatype(self): (rdflib_cuba["datatypes/VECTOR-INT-2-2"], rdflib.RDF.type, rdflib.RDFS.Datatype)}) - def test_check_default_rel(self): - self.parser._check_default_rel("relationshipB", - self.ontology_doc["relationshipB"]) + def test_check_default_rel_flag_on_entity(self): + self.parser._check_default_rel_flag_on_entity( + "relationshipB", + self.ontology_doc["relationshipB"] + ) self.assertEqual(list(self.graph), []) - self.parser._check_default_rel("relationshipA", - self.ontology_doc["relationshipA"]) + self.parser._check_default_rel_flag_on_entity( + "relationshipA", + self.ontology_doc["relationshipA"] + ) self.assertEqual(set(self.graph), {( self.parser._get_iri(), rdflib_cuba._default_rel, self.parser._get_iri("relationshipA") From a5f846327fee4806299bec443cc756b85aa3fdd0 Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Thu, 13 Aug 2020 08:45:03 +0200 Subject: [PATCH 22/22] utils.deserialize will return a list, although the input of serialize was a single CUDS object Fixes #485 (#486) * utils.deserialize will return a list, although the input of serialize was a single CUDS object Fixes #485 * minor --- osp/core/utils/general.py | 14 ++++++++++++-- tests/test_utils.py | 8 ++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/osp/core/utils/general.py b/osp/core/utils/general.py index 82f538ca..ad5b3568 100644 --- a/osp/core/utils/general.py +++ b/osp/core/utils/general.py @@ -126,7 +126,8 @@ def serialize(cuds_object, rel=cuba.activeRelationship, return result -def deserialize(json_doc, session=None, buffer_context=None): +def deserialize(json_doc, session=None, buffer_context=None, + only_return_first_element=True): """Deserialize the given json objects (to CUDS). Will add the CUDS objects to the buffers. @@ -138,6 +139,12 @@ def deserialize(json_doc, session=None, buffer_context=None): buffer_context (BufferContext): Whether to add the objects to the buffers of the user or the engine. Default is equivalent of the user creating the CUDS objects by hand. + only_return_first_element (bool): When the json doc is a list, + whether to return only the first element. The reason + for this is that the result of serializing a single cuds + object using `serialize()` is a list. Having this flag set to True, + the result of deserializing this list will be the input + CUDS object of serialize, as expected. Returns: Any: The deserialized data. Can be CUDS. @@ -150,11 +157,14 @@ def deserialize(json_doc, session=None, buffer_context=None): json_doc = json.loads(json_doc) session = session or Cuds._session buffer_context = buffer_context or BufferContext.USER - return _deserialize( + deserialized = _deserialize( json_obj=json_doc, session=session, buffer_context=buffer_context ) + if isinstance(deserialized, list) and only_return_first_element: + return deserialized[0] + return deserialized def remove_cuds_object(cuds_object): diff --git a/tests/test_utils.py b/tests/test_utils.py index 378d6026..5423893d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -244,12 +244,16 @@ def test_deserialize(self): self.assertEqual(result.name, "Peter") self.assertEqual(result.age, 23) result = deserialize([CUDS_DICT]) + self.assertTrue(result.is_a(city.Citizen)) + self.assertEqual(result.name, "Peter") + self.assertEqual(result.age, 23) + result = deserialize([CUDS_DICT], only_return_first_element=False) self.assertEqual(len(result), 1) self.assertTrue(result[0].is_a(city.Citizen)) self.assertEqual(result[0].name, "Peter") self.assertEqual(result[0].age, 23) - self.assertEqual(CUDS_LIST, serialize(deserialize(CUDS_LIST)[0], - json_dumps=False)) + self.assertEqual(CUDS_LIST, + json.loads(serialize(deserialize(CUDS_LIST)))) def test_serialize(self): c = branch(