From fd5db49c8a6f2f2096725b31fd75e49bd18196c2 Mon Sep 17 00:00:00 2001 From: cmungall Date: Sun, 23 Apr 2023 13:44:19 +0200 Subject: [PATCH 01/15] Adding MultiFileTransformer --- src/linkml_transformer/compiler/__init__.py | 1 + src/linkml_transformer/compiler/compiler.py | 4 +- .../transformer/inference.py | 28 ++++ .../transformer/transformer.py | 18 ++- .../utils/multi_file_transformer.py | 152 ++++++++++++++++++ .../examples/measurements/instructions.yaml | 6 + .../transform/qv-to-scalar.transform.yaml | 2 +- .../test_transformer_examples.py | 27 +++- 8 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 src/linkml_transformer/transformer/inference.py create mode 100644 src/linkml_transformer/utils/multi_file_transformer.py create mode 100644 tests/input/examples/measurements/instructions.yaml diff --git a/src/linkml_transformer/compiler/__init__.py b/src/linkml_transformer/compiler/__init__.py index e69de29..3bf3bad 100644 --- a/src/linkml_transformer/compiler/__init__.py +++ b/src/linkml_transformer/compiler/__init__.py @@ -0,0 +1 @@ +"""Compiles schemas into other transformation frameworks.""" \ No newline at end of file diff --git a/src/linkml_transformer/compiler/compiler.py b/src/linkml_transformer/compiler/compiler.py index eb86149..01a97d5 100644 --- a/src/linkml_transformer/compiler/compiler.py +++ b/src/linkml_transformer/compiler/compiler.py @@ -1,4 +1,4 @@ -from abc import ABC +from abc import ABC, abstractmethod from dataclasses import dataclass from linkml_runtime import SchemaView @@ -11,7 +11,7 @@ @dataclass class Compiler(ABC): """ - Base class for all compiler. + Base class for all compilers. A compiler will compile a transformation specification into an alternative representation. diff --git a/src/linkml_transformer/transformer/inference.py b/src/linkml_transformer/transformer/inference.py new file mode 100644 index 0000000..4f4c351 --- /dev/null +++ b/src/linkml_transformer/transformer/inference.py @@ -0,0 +1,28 @@ +import re + +from linkml_runtime import SchemaView + +from linkml_transformer.datamodel.transformer_model import TransformationSpecification, SlotDerivation + + +def induce_missing_values(specification: TransformationSpecification, source_schemaview: SchemaView): + """ + Infer missing values in a specification. + + Currently only uses copy directives. + + :param specification: + :param source_schemaview: + :return: + """ + for cd in specification.class_derivations.values(): + src_cls_name = cd.populated_from + src_cls = source_schemaview.get_class(src_cls_name) + for slot_match, directive in cd.copy_directives.items(): + for sn in source_schemaview.class_slots(src_cls.name): + if sn in cd.slot_derivations: + continue + #if slot_match == "*" or re.match(slot_match, sn): + # sd = SlotDerivation(name=sn, populated_from=sn) + # print(f"Adding {src_cls_name} . {sd}") + # cd.slot_derivations[sd.name] = sd diff --git a/src/linkml_transformer/transformer/transformer.py b/src/linkml_transformer/transformer/transformer.py index d91d0e7..2d76ad5 100644 --- a/src/linkml_transformer/transformer/transformer.py +++ b/src/linkml_transformer/transformer/transformer.py @@ -1,5 +1,6 @@ import logging from abc import ABC +from copy import deepcopy from dataclasses import dataclass from pathlib import Path from types import ModuleType @@ -12,11 +13,13 @@ from linkml_transformer.datamodel.transformer_model import ( ClassDerivation, TransformationSpecification, SlotDerivation, CollectionType) +from linkml_transformer.transformer.inference import induce_missing_values logger = logging.getLogger(__name__) OBJECT_TYPE = Union[Dict[str, Any], BaseModel, YAMLRoot] +"""An object can be a plain python dict, a pydantic object, or a linkml YAMLRoot""" @dataclass @@ -37,6 +40,9 @@ class Transformer(ABC): specification: TransformationSpecification = None """A specification of how to generate target objects from source objects.""" + _derived_specification: TransformationSpecification = None + """A specification with inferred missing values.""" + target_schemaview: Optional[SchemaView] = None """A view over the schema describing the output/target object.""" @@ -71,8 +77,18 @@ def load_transformer_specification(self, path: Union[str, Path]): """ self.specification = yaml_loader.load(str(path), TransformationSpecification) + @property + def derived_specification(self) -> Optional[TransformationSpecification]: + if self._derived_specification is None: + if self.specification is None: + return None + self._derived_specification = deepcopy(self.specification) + induce_missing_values(self._derived_specification, self.source_schemaview) + return self._derived_specification + + def _get_class_derivation(self, target_class_name) -> ClassDerivation: - spec = self.specification + spec = self.derived_specification matching_tgt_class_derivs = [ deriv for deriv in spec.class_derivations.values() diff --git a/src/linkml_transformer/utils/multi_file_transformer.py b/src/linkml_transformer/utils/multi_file_transformer.py new file mode 100644 index 0000000..fbf21d7 --- /dev/null +++ b/src/linkml_transformer/utils/multi_file_transformer.py @@ -0,0 +1,152 @@ +"""Iterate through all examples in a folder testing them for validity. + +""" +import glob +import json +import logging +import os +import re +import sys +from dataclasses import dataclass, field +from io import StringIO +from pathlib import Path +from types import ModuleType +from typing import Union, Any, Mapping, Optional, List, TextIO + +import click +import yaml +from linkml_runtime import SchemaView +from linkml_runtime.dumpers import json_dumper, rdflib_dumper, yaml_dumper +from linkml_runtime.linkml_model import ElementName +from linkml_runtime.utils.formatutils import camelcase +from pydantic import BaseModel + +from linkml_transformer.transformer.object_transformer import ObjectTransformer + + +class Step(BaseModel): + source_data: str + target_data: Optional[str] = None + +class Transformation(BaseModel): + description: Optional[str] = None + source_schema: str + transformation_specification: str + steps: List[Step] = None + +class Instructions(BaseModel): + description: Optional[str] = None + transformations: List[Transformation] = [] + + + +@dataclass +class MultiFileTransformer: + """ + Processes a collection of inputs in one folder. + + Assumes folder structures: + + - {package} + - source/{source_schema}.yaml :: the schema to transform from + - transform/{transform_spec}.transform.yaml :: mapping spec + - data/{SourceClassName}-{LocalId}.yaml :: data to transform + - target/{SourceClassName}-{LocalId}.yaml :: expected output data + """ + + source_schema_directory_base: str = field(default_factory=lambda: "source") + transform_specification_directory_base: str = field(default_factory=lambda: "transform") + source_data_directory_base: str = field(default_factory=lambda: "data") + target_data_directory_base: str = field(default_factory=lambda: "target") + + input_formats: Optional[List[str]] = field(default_factory=lambda: ['yaml']) + """Expected formats for input data""" + + output_formats: Optional[List[str]] = field(default_factory=lambda: ['yaml']) + + prefix_map: Optional[Mapping[str, str]] = None + """Custom prefix map, for emitting RDF/turtle.""" + + def process_directory(self, root_directory: Union[str, Path], **kwargs): + """ + Process all transformations in a directory. + + :param root_directory: + :return: + """ + instructions = self.infer_instructions(root_directory) + return self.process_instructions(instructions, root_directory, **kwargs) + + def infer_instructions(self, root_directory: Union[str, Path]) -> Instructions: + """ + Infer instructions from either explicit yaml or directory layout. + """ + if isinstance(root_directory, str): + root_directory = Path(root_directory) + if not root_directory.exists(): + raise ValueError(f"No such directory {root_directory}") + instructions_file = root_directory / "instructions.yaml" + if instructions_file.exists(): + with open(instructions_file) as file: + return Instructions(**yaml.safe_load(file)) + source_schema_directory = root_directory / self.source_schema_directory_base + if not source_schema_directory.exists(): + raise ValueError(f"Expected {source_schema_directory}") + transform_specification_directory = root_directory / self.transform_specification_directory_base + if not transform_specification_directory.exists(): + raise ValueError(f"Expected {transform_specification_directory}") + input_schemas = glob.glob(os.path.join(str(source_schema_directory), "*.yaml")) + if not input_schemas: + raise ValueError(f"Expected schemas in {source_schema_directory}") + instructions = Instructions(description="auto-inducted") + for input_schema in input_schemas: + transform_files = glob.glob(os.path.join(str(transform_specification_directory), "*.transform.yaml")) + for transform_file in transform_files: + tr = Transformation(source_schema=input_schema, + transform_specification=transform_file) + target_schema_base = None + if len(input_schemas) != 1: + # resolve which schema the transform applies to + matches = re.match(r'^(\w+)-to-(\w+)\.', transform_file) + if not matches: + raise ValueError(f"Ambiguous: {transform_file}") + src, target_schema_base = matches.group(1, 2) + if src not in input_schema: + continue + instructions.transformations.append(tr) + data_files = glob.glob(os.path.join(str(self.source_data_directory_base), "*.yaml")) + for data_file in data_files: + if target_schema_base and target_schema_base not in data_file: + continue + step = Step(source_data=data_file) + tr.steps.append(step) + if not tr.steps: + raise ValueError(f"Could not infer steps from {data_files}") + return instructions + + def process_instructions(self, instructions: Instructions, root_directory: Optional[Union[str, Path]], test_mode=False): + """ + Process a set of instructions to transform one or more files. + + :param instructions: + :param root_directory: + :param test_mode: + :return: + """ + if isinstance(root_directory, str): + directory = Path(root_directory) + logging.info(f"Processing: {instructions.description}") + for tr in instructions.transformations: + source_schema_path = str(root_directory / tr.source_schema) + sv = SchemaView(source_schema_path) + transformer = ObjectTransformer() + transformer.load_source_schema(source_schema_path) + tr_path = str(root_directory / tr.transformation_specification) + print(tr_path) + transformer.load_transformer_specification(tr_path) + for step in tr.steps: + input_obj = yaml.safe_load(open(str(root_directory / step.source_data))) + print(input_obj) + target_obj = transformer.transform(input_obj) + print(target_obj) + diff --git a/tests/input/examples/measurements/instructions.yaml b/tests/input/examples/measurements/instructions.yaml new file mode 100644 index 0000000..c839a65 --- /dev/null +++ b/tests/input/examples/measurements/instructions.yaml @@ -0,0 +1,6 @@ +description: Examples of transforming measurements +transformations: + - source_schema: source/quantity_value.yaml + transformation_specification: transform/qv-to-scalar.transform.yaml + steps: + - source_data: data/PersonQuantityValue-001.yaml diff --git a/tests/input/examples/measurements/transform/qv-to-scalar.transform.yaml b/tests/input/examples/measurements/transform/qv-to-scalar.transform.yaml index b060adc..315e2b8 100644 --- a/tests/input/examples/measurements/transform/qv-to-scalar.transform.yaml +++ b/tests/input/examples/measurements/transform/qv-to-scalar.transform.yaml @@ -3,8 +3,8 @@ class_derivations: Person: populated_from: Person slot_derivations: + id: height_in_cm: - #expr: "{height}.value if {height}.unit == 'cm' else ''" expr: "case( ({height}.unit == 'cm', {height}.value), (True, NULL) )" copy_directives: "*": diff --git a/tests/test_transformer/test_transformer_examples.py b/tests/test_transformer/test_transformer_examples.py index 5832f24..5ab17f0 100644 --- a/tests/test_transformer/test_transformer_examples.py +++ b/tests/test_transformer/test_transformer_examples.py @@ -4,6 +4,7 @@ from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.utils.dynamic_object import dynamic_object +from linkml_transformer.utils.multi_file_transformer import MultiFileTransformer from tests import EXAMPLE_DIR EXAMPLES = [ @@ -14,7 +15,16 @@ class TransformerExamplesTestCase(unittest.TestCase): """ - Tests ObjectTransformer using examples + Tests ObjectTransformer using examples. + + Assumes folder structures: + + - input/examples + - {package} + - source/{source_schema}.yaml :: the schema to transform from + - transform/{transform_spec}.transform.yaml :: mapping spec + - data/{SourceClassName}-{LocalId}.yaml :: data to transform + - target/{SourceClassName}-{LocalId}.yaml :: expected output data """ def test_examples(self): @@ -33,3 +43,18 @@ def test_examples(self): target_obj = tr.transform(input_obj) # target_obj = dynamic_object(target_obj, "Person") self.assertEqual(expected, target_obj["height_in_cm"]) + + def test_all(self): + """ + Iterates through all examples + :return: + """ + mft = MultiFileTransformer() + dirs = ["measurements"] + for dir in dirs: + full_dir = EXAMPLE_DIR / dir + instructions = mft.infer_instructions(full_dir) + print(instructions) + obj = mft.process_instructions(instructions, full_dir) + print(obj) + From 99c79028af180b93f26f9fd1c76ab8886dfdc834 Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 25 Apr 2023 09:49:53 +0200 Subject: [PATCH 02/15] Adding type coercion. Fixes #2 --- .../transformer/object_transformer.py | 22 +++++++++--- .../transformer/transformer.py | 35 +++++++++++++++++- .../type_coercion/source/type_coercion.yaml | 36 +++++++++++++++++++ .../transform/type_coercion.transform.yaml | 32 +++++++++++++++++ 4 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 tests/input/examples/type_coercion/source/type_coercion.yaml create mode 100644 tests/input/examples/type_coercion/transform/type_coercion.transform.yaml diff --git a/src/linkml_transformer/transformer/object_transformer.py b/src/linkml_transformer/transformer/object_transformer.py index d7725d8..654103e 100644 --- a/src/linkml_transformer/transformer/object_transformer.py +++ b/src/linkml_transformer/transformer/object_transformer.py @@ -59,7 +59,8 @@ def transform( self, source_obj: OBJECT_TYPE, source_type: str = None, - ) -> DICT_OBJ: + target_type: str = None, + ) -> Union[DICT_OBJ, Any]: """ Transform a source object into a target object. @@ -71,7 +72,17 @@ def transform( if source_type is None: [source_type] = [c.name for c in sv.all_classes().values() if c.tree_root] if source_type in sv.all_types(): - # TODO: type derivations + if target_type: + if target_type == "string": + return str(source_obj) + elif target_type == "integer": + return int(source_obj) + elif target_type == "float" or target_type == "double": + return float(source_obj) + elif target_type == "uri": + return self.expand_curie(source_obj) + elif target_type == "curie": + return self.compress_uri(source_obj) return source_obj if source_type in sv.all_enums(): # TODO: enum derivations @@ -120,16 +131,17 @@ def transform( source_class_slot = sv.induced_slot(slot_derivation.name, source_type) v = source_obj.get(slot_derivation.name, None) if source_class_slot and v is not None: + target_range = slot_derivation.range source_class_slot_range = source_class_slot.range if source_class_slot.multivalued: if isinstance(v, list): - v = [self.transform(v1, source_class_slot_range) for v1 in v] + v = [self.transform(v1, source_class_slot_range, target_range) for v1 in v] elif isinstance(v, dict): - v = [self.transform(v1, source_class_slot_range) for v1 in v] + v = [self.transform(v1, source_class_slot_range, target_range) for v1 in v] else: v = [v] else: - v = self.transform(v, source_class_slot_range) + v = self.transform(v, source_class_slot_range, target_range) if self._coerce_to_multivalued(slot_derivation, class_deriv) and v is not None and not isinstance(v, list): v = [v] if self._coerce_to_singlevalued(slot_derivation, class_deriv) and isinstance(v, list): diff --git a/src/linkml_transformer/transformer/transformer.py b/src/linkml_transformer/transformer/transformer.py index 2d76ad5..fd027ac 100644 --- a/src/linkml_transformer/transformer/transformer.py +++ b/src/linkml_transformer/transformer/transformer.py @@ -6,10 +6,14 @@ from types import ModuleType from typing import Any, Dict, Optional, Type, Union +import yaml from linkml_runtime import SchemaView from linkml_runtime.loaders import yaml_loader +from linkml_runtime.processing.referencevalidator import ReferenceValidator +from linkml_runtime.utils.introspection import package_schemaview from linkml_runtime.utils.yamlutils import YAMLRoot from pydantic import BaseModel +from curies import Converter from linkml_transformer.datamodel.transformer_model import ( ClassDerivation, TransformationSpecification, SlotDerivation, CollectionType) @@ -49,6 +53,11 @@ class Transformer(ABC): target_module: Optional[ModuleType] = None """The python module which the target object should conform to.""" + prefix_map: Optional[Dict[str, str]] = None + """Additional prefixes""" + + _curie_converter: Converter = None + def transform(self, obj: OBJECT_TYPE, source_type: str = None) -> OBJECT_TYPE: """ Transform source object into an instance of the target class. @@ -75,7 +84,13 @@ def load_transformer_specification(self, path: Union[str, Path]): :param path: :return: """ - self.specification = yaml_loader.load(str(path), TransformationSpecification) + # self.specification = yaml_loader.load(str(path), TransformationSpecification) + with open(path) as f: + obj = yaml.safe_load(f) + normalizer = ReferenceValidator(package_schemaview("linkml_transformer.datamodel.transformer_model")) + normalizer.expand_all = True + obj = normalizer.normalize(obj) + self.specification = TransformationSpecification(**obj) @property def derived_specification(self) -> Optional[TransformationSpecification]: @@ -122,3 +137,21 @@ def _coerce_to_singlevalued(self, slot_derivation: SlotDerivation, class_derivat if not slot.multivalued: return True return False + + @property + def curie_converter(self) -> Converter: + if not self._curie_converter: + self._curie_converter = Converter([]) + for prefix in self.source_schemaview.schema.prefixes.values(): + self._curie_converter.add_prefix(prefix.prefix_prefix, prefix.prefix_reference) + for prefix in self.specification.prefixes.values(): + self._curie_converter.add_prefix(prefix.key, prefix.value) + return self._curie_converter + + def expand_curie(self, curie: str) -> str: + return self.curie_converter.expand(curie) + + def compress_uri(self, uri: str) -> str: + return self.curie_converter.compress(uri) + + diff --git a/tests/input/examples/type_coercion/source/type_coercion.yaml b/tests/input/examples/type_coercion/source/type_coercion.yaml new file mode 100644 index 0000000..e7d447a --- /dev/null +++ b/tests/input/examples/type_coercion/source/type_coercion.yaml @@ -0,0 +1,36 @@ +id: https://w3id.org/linkml/examples/type_coercion +name: quantity_value +imports: + - linkml:types +prefixes: + tc: https://w3id.org/linkml/examples/type_coercion/ + linkml: https://w3id.org/linkml/ + example: https://example.org/ + P: https://example.org/person/ +default_prefix: tc +default_range: string + +classes: + + MyRecord: + tree_root: true + attributes: + id: + identifier: true + range: uriorcurie + id_as_string: + range: string + url: + range: uri + url_as_string: + range: string + name: + range: string + age_as_integer: + range: integer + age_as_float: + range: float + age_as_string: + range: string + + diff --git a/tests/input/examples/type_coercion/transform/type_coercion.transform.yaml b/tests/input/examples/type_coercion/transform/type_coercion.transform.yaml new file mode 100644 index 0000000..75162c5 --- /dev/null +++ b/tests/input/examples/type_coercion/transform/type_coercion.transform.yaml @@ -0,0 +1,32 @@ +id: qv-to-scalar +class_derivations: + MyRecord: + populated_from: MyRecord + slot_derivations: + id: + id_as_string: + range: string + populated_from: id + id_as_url: + range: uri + populated_from: id + url: + url_as_string: + range: string + populated_from: url + url_as_curie: + range: curie + populated_from: url + name: + range: string + age_as_integer: + range: integer + age_as_float: + range: float + populated_from: age_as_integer + age_as_string: + range: string + populated_from: age_as_integer + copy_directives: + "*": + copy_all: true From 84c228b6bf5c9e910250d2c580485c399ec8d7a9 Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 25 Apr 2023 09:52:40 +0200 Subject: [PATCH 03/15] refactor and lint --- Makefile | 3 + src/linkml_transformer/compiler/__init__.py | 2 +- .../datamodel/transformer_model.py | 790 ++++++------------ .../datamodel/transformer_model.yaml | 21 +- .../schema_mapper/schema_mapper.py | 46 +- .../transformer/inference.py | 9 +- .../transformer/object_transformer.py | 21 +- .../transformer/transformer.py | 24 +- src/linkml_transformer/utils/inverter.py | 7 +- .../utils/multi_file_transformer.py | 129 ++- tests/input/examples/README.md | 31 + .../input/examples/biolink/data/gene-001.yaml | 3 + .../biolink/source/biolink-model.yaml | 1 - .../biolink-example-profile.transform.yaml | 10 +- .../examples/measurements/instructions.yaml | 4 + .../personinfo_basic/model/agent_model.py | 38 +- .../test_object_transformer.py | 21 +- .../test_transformer_examples.py | 35 +- 18 files changed, 551 insertions(+), 644 deletions(-) create mode 100644 tests/input/examples/README.md create mode 100644 tests/input/examples/biolink/data/gene-001.yaml diff --git a/Makefile b/Makefile index 7e93856..41dfa58 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,9 @@ all: gen-project gendoc %.yaml: gen-project deploy: all mkd-gh-deploy +src/linkml_transformer/datamodel/transformer_model.py: src/linkml_transformer/datamodel/transformer_model.yaml + $(RUN) gen-pydantic $< > $@.tmp && mv $@.tmp $@ + # generates all project files gen-project: $(PYMODEL) $(RUN) gen-project -d $(DEST) $(SOURCE_SCHEMA_PATH) && mv $(DEST)/*.py $(PYMODEL) diff --git a/src/linkml_transformer/compiler/__init__.py b/src/linkml_transformer/compiler/__init__.py index 3bf3bad..1445353 100644 --- a/src/linkml_transformer/compiler/__init__.py +++ b/src/linkml_transformer/compiler/__init__.py @@ -1 +1 @@ -"""Compiles schemas into other transformation frameworks.""" \ No newline at end of file +"""Compiles schemas into other transformation frameworks.""" diff --git a/src/linkml_transformer/datamodel/transformer_model.py b/src/linkml_transformer/datamodel/transformer_model.py index 00ada18..7013aa5 100644 --- a/src/linkml_transformer/datamodel/transformer_model.py +++ b/src/linkml_transformer/datamodel/transformer_model.py @@ -1,614 +1,314 @@ -# Auto generated from transformer_model.yaml by pythongen.py version: 0.9.0 -# Generation date: 2023-03-07T16:06:07 -# Schema: transformer -# -# id: https://w3id.org/linkml/transformer -# description: Datamodel for LinkML schema transformations. A transformer generates instances of a *target* data -# model from instances of a *source* data model. This transformation process is guided by a -# *TransformationSpecification*. The specification is independent of any one method for transforming -# data. It allows different approaches, including: - direct implementation, transforming python or -# json objects - translation of the specification into SQL commands, to operate on relations - -# translation of the specification into SPARQL CONSTRUCTs, to operate on triples - translation into -# another specification language, such as R2RML -# license: https://creativecommons.org/publicdomain/zero/1.0/ - -import dataclasses -import sys -import re -from jsonasobj2 import JsonObj, as_dict -from typing import Optional, List, Union, Dict, ClassVar, Any -from dataclasses import dataclass -from linkml_runtime.linkml_model.meta import EnumDefinition, PermissibleValue, PvFormulaOptions - -from linkml_runtime.utils.slot import Slot -from linkml_runtime.utils.metamodelcore import empty_list, empty_dict, bnode -from linkml_runtime.utils.yamlutils import YAMLRoot, extended_str, extended_float, extended_int -from linkml_runtime.utils.dataclass_extensions_376 import dataclasses_init_fn_with_kwargs -from linkml_runtime.utils.formatutils import camelcase, underscore, sfx -from linkml_runtime.utils.enumerations import EnumDefinitionImpl -from rdflib import Namespace, URIRef -from linkml_runtime.utils.curienamespace import CurieNamespace -from linkml_runtime.linkml_model.types import Boolean, String -from linkml_runtime.utils.metamodelcore import Bool - -metamodel_version = "1.7.0" -version = None - -# Overwrite dataclasses _init_fn to add **kwargs in __init__ -dataclasses._init_fn = dataclasses_init_fn_with_kwargs - -# Namespaces -LINKML = CurieNamespace('linkml', 'https://w3id.org/linkml/') -LINKMLTR = CurieNamespace('linkmltr', 'https://w3id.org/linkml/transformer/') -DEFAULT_ = LINKMLTR - - -# Types - -# Class references -class TransformationSpecificationId(extended_str): - pass - - -class ElementDerivationName(extended_str): - pass - - -class ClassDerivationName(ElementDerivationName): - pass - +from __future__ import annotations -class AliasedClassAlias(extended_str): - pass +from datetime import date, datetime +from enum import Enum +from typing import Any, Dict, List, Literal, Optional, Union +from linkml_runtime.linkml_model import Decimal +from pydantic import BaseModel as BaseModel +from pydantic import Field -class SlotDerivationName(ElementDerivationName): - pass +metamodel_version = "None" +version = "None" -class EnumDerivationName(ElementDerivationName): - pass +class WeakRefShimBaseModel(BaseModel): + __slots__ = "__weakref__" -class PermissibleValueDerivationName(ElementDerivationName): +class ConfiguredBaseModel( + WeakRefShimBaseModel, + validate_assignment=True, + validate_all=True, + underscore_attrs_are_private=True, + extra="forbid", + arbitrary_types_allowed=True, +): pass -class PrefixDerivationName(ElementDerivationName): - pass +class CollectionType(str, Enum): + SingleValued = "SingleValued" + MultiValued = "MultiValued" + MultiValuedList = "MultiValuedList" + MultiValuedDict = "MultiValuedDict" -class KeyValKey(extended_str): - pass - - -class CopyDirectiveElementName(extended_str): - pass - - -Any = Any - -@dataclass -class TransformationSpecification(YAMLRoot): +class TransformationSpecification(ConfiguredBaseModel): """ A collection of mappings between source and target classes """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.TransformationSpecification - class_class_curie: ClassVar[str] = "linkmltr:TransformationSpecification" - class_name: ClassVar[str] = "TransformationSpecification" - class_model_uri: ClassVar[URIRef] = LINKMLTR.TransformationSpecification - - id: Union[str, TransformationSpecificationId] = None - title: Optional[str] = None - prefixes: Optional[str] = None - source_schema: Optional[str] = None - target_schema: Optional[str] = None - class_derivations: Optional[Union[Dict[Union[str, ClassDerivationName], Union[dict, "ClassDerivation"]], List[Union[dict, "ClassDerivation"]]]] = empty_dict() - enum_derivations: Optional[Union[Dict[Union[str, EnumDerivationName], Union[dict, "EnumDerivation"]], List[Union[dict, "EnumDerivation"]]]] = empty_dict() - slot_derivations: Optional[Union[Dict[Union[str, SlotDerivationName], Union[dict, "SlotDerivation"]], List[Union[dict, "SlotDerivation"]]]] = empty_dict() - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.id): - self.MissingRequiredField("id") - if not isinstance(self.id, TransformationSpecificationId): - self.id = TransformationSpecificationId(self.id) - - if self.title is not None and not isinstance(self.title, str): - self.title = str(self.title) - if self.prefixes is not None and not isinstance(self.prefixes, str): - self.prefixes = str(self.prefixes) - - if self.source_schema is not None and not isinstance(self.source_schema, str): - self.source_schema = str(self.source_schema) - - if self.target_schema is not None and not isinstance(self.target_schema, str): - self.target_schema = str(self.target_schema) - - self._normalize_inlined_as_dict(slot_name="class_derivations", slot_type=ClassDerivation, key_name="name", keyed=True) - - self._normalize_inlined_as_dict(slot_name="enum_derivations", slot_type=EnumDerivation, key_name="name", keyed=True) - - self._normalize_inlined_as_dict(slot_name="slot_derivations", slot_type=SlotDerivation, key_name="name", keyed=True) - - super().__post_init__(**kwargs) + id: Optional[str] = Field( + None, description="""Unique identifier for this transformation specification""" + ) + title: Optional[str] = Field( + None, + description="""human readable title for this transformation specification""", + ) + prefixes: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, description="""maps prefixes to URL expansions""" + ) + source_schema: Optional[str] = Field( + None, + description="""name of the schema that describes the source (input) objects""", + ) + target_schema: Optional[str] = Field( + None, + description="""name of the schema that describes the target (output) objects""", + ) + class_derivations: Optional[Dict[str, ClassDerivation]] = Field( + default_factory=dict, + description="""Instructions on how to derive a set of classes in the target schema from classes in the source schema.""", + ) + enum_derivations: Optional[Dict[str, EnumDerivation]] = Field( + default_factory=dict, + description="""Instructions on how to derive a set of enums in the target schema""", + ) + slot_derivations: Optional[Dict[str, SlotDerivation]] = Field( + default_factory=dict, + description="""Instructions on how to derive a set of top level slots in the target schema""", + ) -@dataclass -class ElementDerivation(YAMLRoot): +class ElementDerivation(ConfiguredBaseModel): """ - An abstract grouping for classes that provide a specification of how to derive a target element from a source - element. + An abstract grouping for classes that provide a specification of how to derive a target element from a source element. """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.ElementDerivation - class_class_curie: ClassVar[str] = "linkmltr:ElementDerivation" - class_name: ClassVar[str] = "ElementDerivation" - class_model_uri: ClassVar[URIRef] = LINKMLTR.ElementDerivation - - name: Union[str, ElementDerivationName] = None - copy_directives: Optional[Union[Dict[Union[str, CopyDirectiveElementName], Union[dict, "CopyDirective"]], List[Union[dict, "CopyDirective"]]]] = empty_dict() - is_a: Optional[Union[str, ElementDerivationName]] = None - mixins: Optional[Union[Dict[Union[str, ElementDerivationName], Union[dict, "ElementDerivation"]], List[Union[dict, "ElementDerivation"]]]] = empty_dict() - value_mappings: Optional[Union[Dict[Union[str, KeyValKey], Union[dict, "KeyVal"]], List[Union[dict, "KeyVal"]]]] = empty_dict() - expression_to_value_mappings: Optional[Union[Dict[Union[str, KeyValKey], Union[dict, "KeyVal"]], List[Union[dict, "KeyVal"]]]] = empty_dict() - expression_to_expression_mappings: Optional[Union[Dict[Union[str, KeyValKey], Union[dict, "KeyVal"]], List[Union[dict, "KeyVal"]]]] = empty_dict() - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.name): - self.MissingRequiredField("name") - if not isinstance(self.name, ElementDerivationName): - self.name = ElementDerivationName(self.name) - - self._normalize_inlined_as_dict(slot_name="copy_directives", slot_type=CopyDirective, key_name="element_name", keyed=True) - - if self.is_a is not None and not isinstance(self.is_a, ElementDerivationName): - self.is_a = ElementDerivationName(self.is_a) - - self._normalize_inlined_as_dict(slot_name="mixins", slot_type=ElementDerivation, key_name="name", keyed=True) - - self._normalize_inlined_as_dict(slot_name="value_mappings", slot_type=KeyVal, key_name="key", keyed=True) - - self._normalize_inlined_as_dict(slot_name="expression_to_value_mappings", slot_type=KeyVal, key_name="key", keyed=True) - - self._normalize_inlined_as_dict(slot_name="expression_to_expression_mappings", slot_type=KeyVal, key_name="key", keyed=True) - super().__post_init__(**kwargs) + name: str = Field(None, description="""Name of the element in the target schema""") + copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) + overrides: Optional[Any] = Field( + None, description="""overrides source schema slots""" + ) + is_a: Optional[str] = Field(None) + mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) + value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table that is applied directly to mappings, in order of precedence""", + ) + expression_to_value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys are expressions""", + ) + expression_to_expression_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys and values are expressions""", + ) -@dataclass class ClassDerivation(ElementDerivation): """ A specification of how to derive a target class from a source class. """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.ClassDerivation - class_class_curie: ClassVar[str] = "linkmltr:ClassDerivation" - class_name: ClassVar[str] = "ClassDerivation" - class_model_uri: ClassVar[URIRef] = LINKMLTR.ClassDerivation - - name: Union[str, ClassDerivationName] = None - populated_from: Optional[str] = None - joins: Optional[Union[Dict[Union[str, AliasedClassAlias], Union[dict, "AliasedClass"]], List[Union[dict, "AliasedClass"]]]] = empty_dict() - slot_derivations: Optional[Union[Dict[Union[str, SlotDerivationName], Union[dict, "SlotDerivation"]], List[Union[dict, "SlotDerivation"]]]] = empty_dict() - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.name): - self.MissingRequiredField("name") - if not isinstance(self.name, ClassDerivationName): - self.name = ClassDerivationName(self.name) - - if self.populated_from is not None and not isinstance(self.populated_from, str): - self.populated_from = str(self.populated_from) - - self._normalize_inlined_as_dict(slot_name="joins", slot_type=AliasedClass, key_name="alias", keyed=True) - - self._normalize_inlined_as_dict(slot_name="slot_derivations", slot_type=SlotDerivation, key_name="name", keyed=True) - - super().__post_init__(**kwargs) + populated_from: Optional[str] = Field( + None, description="""Name of the class in the source schema""" + ) + joins: Optional[Dict[str, AliasedClass]] = Field( + default_factory=dict, + description="""Additional classes to be joined to derive instances of the target class""", + ) + slot_derivations: Optional[Dict[str, SlotDerivation]] = Field(default_factory=dict) + name: str = Field(None, description="""Name of the element in the target schema""") + copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) + overrides: Optional[Any] = Field( + None, description="""overrides source schema slots""" + ) + is_a: Optional[str] = Field(None) + mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) + value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table that is applied directly to mappings, in order of precedence""", + ) + expression_to_value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys are expressions""", + ) + expression_to_expression_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys and values are expressions""", + ) -@dataclass -class AliasedClass(YAMLRoot): +class AliasedClass(ConfiguredBaseModel): """ alias-class key value pairs for classes """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.AliasedClass - class_class_curie: ClassVar[str] = "linkmltr:AliasedClass" - class_name: ClassVar[str] = "AliasedClass" - class_model_uri: ClassVar[URIRef] = LINKMLTR.AliasedClass - - alias: Union[str, AliasedClassAlias] = None - class_named: Optional[str] = None - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.alias): - self.MissingRequiredField("alias") - if not isinstance(self.alias, AliasedClassAlias): - self.alias = AliasedClassAlias(self.alias) - - if self.class_named is not None and not isinstance(self.class_named, str): - self.class_named = str(self.class_named) - - super().__post_init__(**kwargs) + alias: str = Field(None, description="""name of the class to be aliased""") + class_named: Optional[str] = Field( + None, description="""local alias for the class""" + ) -@dataclass class SlotDerivation(ElementDerivation): """ A specification of how to derive the value of a target slot from a source slot """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.SlotDerivation - class_class_curie: ClassVar[str] = "linkmltr:SlotDerivation" - class_name: ClassVar[str] = "SlotDerivation" - class_model_uri: ClassVar[URIRef] = LINKMLTR.SlotDerivation - - name: Union[str, SlotDerivationName] = None - populated_from: Optional[str] = None - expr: Optional[str] = None - inverse_of: Optional[Union[dict, "Inverse"]] = None - hide: Optional[Union[bool, Bool]] = None - type_designator: Optional[Union[bool, Bool]] = None - cast_collection_as: Optional[Union[str, "CollectionType"]] = None - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.name): - self.MissingRequiredField("name") - if not isinstance(self.name, SlotDerivationName): - self.name = SlotDerivationName(self.name) - - if self.populated_from is not None and not isinstance(self.populated_from, str): - self.populated_from = str(self.populated_from) - - if self.expr is not None and not isinstance(self.expr, str): - self.expr = str(self.expr) - - if self.inverse_of is not None and not isinstance(self.inverse_of, Inverse): - self.inverse_of = Inverse(**as_dict(self.inverse_of)) - - if self.hide is not None and not isinstance(self.hide, Bool): - self.hide = Bool(self.hide) - - if self.type_designator is not None and not isinstance(self.type_designator, Bool): - self.type_designator = Bool(self.type_designator) - - if self.cast_collection_as is not None and not isinstance(self.cast_collection_as, CollectionType): - self.cast_collection_as = CollectionType(self.cast_collection_as) - - super().__post_init__(**kwargs) + name: str = Field(None, description="""Target slot name""") + populated_from: Optional[str] = Field(None, description="""Source slot name""") + expr: Optional[str] = Field( + None, + description="""An expression to be evaluated on the source object to derive the target slot. Should be specified using the LinkML expression language.""", + ) + range: Optional[str] = Field(None) + inverse_of: Optional[Inverse] = Field( + None, + description="""Used to specify a class-slot tuple that is the inverse of the derived/target slot. This is used primarily for mapping to relational databases or formalisms that do not allow multiple values. The class representing the repeated element has a foreign key slot inserted in that 'back references' the original multivalued slot.""", + ) + hide: Optional[bool] = Field(None, description="""True if this is suppressed""") + type_designator: Optional[bool] = Field(None) + cast_collection_as: Optional[CollectionType] = Field(None) + copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) + overrides: Optional[Any] = Field( + None, description="""overrides source schema slots""" + ) + is_a: Optional[str] = Field(None) + mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) + value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table that is applied directly to mappings, in order of precedence""", + ) + expression_to_value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys are expressions""", + ) + expression_to_expression_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys and values are expressions""", + ) -@dataclass class EnumDerivation(ElementDerivation): """ A specification of how to derive the value of a target enum from a source enum """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.EnumDerivation - class_class_curie: ClassVar[str] = "linkmltr:EnumDerivation" - class_name: ClassVar[str] = "EnumDerivation" - class_model_uri: ClassVar[URIRef] = LINKMLTR.EnumDerivation - - name: Union[str, EnumDerivationName] = None - populated_from: Optional[str] = None - expr: Optional[str] = None - hide: Optional[Union[bool, Bool]] = None - permissible_value_derivations: Optional[Union[Dict[Union[str, PermissibleValueDerivationName], Union[dict, "PermissibleValueDerivation"]], List[Union[dict, "PermissibleValueDerivation"]]]] = empty_dict() - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.name): - self.MissingRequiredField("name") - if not isinstance(self.name, EnumDerivationName): - self.name = EnumDerivationName(self.name) - - if self.populated_from is not None and not isinstance(self.populated_from, str): - self.populated_from = str(self.populated_from) - - if self.expr is not None and not isinstance(self.expr, str): - self.expr = str(self.expr) - if self.hide is not None and not isinstance(self.hide, Bool): - self.hide = Bool(self.hide) - - self._normalize_inlined_as_dict(slot_name="permissible_value_derivations", slot_type=PermissibleValueDerivation, key_name="name", keyed=True) - - super().__post_init__(**kwargs) + name: str = Field(None, description="""Target enum name""") + populated_from: Optional[str] = Field(None, description="""Source enum name""") + expr: Optional[str] = Field( + None, + description="""An expression to be evaluated on the source object to derive the target slot. Should be specified using the LinkML expression language.""", + ) + hide: Optional[bool] = Field(None, description="""True if this is suppressed""") + permissible_value_derivations: Optional[ + Dict[str, PermissibleValueDerivation] + ] = Field( + default_factory=dict, + description="""Instructions on how to derive a set of PVs in the target schema""", + ) + copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) + overrides: Optional[Any] = Field( + None, description="""overrides source schema slots""" + ) + is_a: Optional[str] = Field(None) + mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) + value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table that is applied directly to mappings, in order of precedence""", + ) + expression_to_value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys are expressions""", + ) + expression_to_expression_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys and values are expressions""", + ) -@dataclass class PermissibleValueDerivation(ElementDerivation): """ A specification of how to derive the value of a PV from a source enum """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.PermissibleValueDerivation - class_class_curie: ClassVar[str] = "linkmltr:PermissibleValueDerivation" - class_name: ClassVar[str] = "PermissibleValueDerivation" - class_model_uri: ClassVar[URIRef] = LINKMLTR.PermissibleValueDerivation - - name: Union[str, PermissibleValueDerivationName] = None - expr: Optional[str] = None - populated_from: Optional[str] = None - hide: Optional[Union[bool, Bool]] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.name): - self.MissingRequiredField("name") - if not isinstance(self.name, PermissibleValueDerivationName): - self.name = PermissibleValueDerivationName(self.name) - if self.expr is not None and not isinstance(self.expr, str): - self.expr = str(self.expr) - - if self.populated_from is not None and not isinstance(self.populated_from, str): - self.populated_from = str(self.populated_from) - - if self.hide is not None and not isinstance(self.hide, Bool): - self.hide = Bool(self.hide) - - super().__post_init__(**kwargs) + name: str = Field(None, description="""Target permissible value text""") + expr: Optional[str] = Field(None) + populated_from: Optional[str] = Field( + None, description="""Source permissible value""" + ) + hide: Optional[bool] = Field(None) + copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) + overrides: Optional[Any] = Field( + None, description="""overrides source schema slots""" + ) + is_a: Optional[str] = Field(None) + mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) + value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table that is applied directly to mappings, in order of precedence""", + ) + expression_to_value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys are expressions""", + ) + expression_to_expression_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys and values are expressions""", + ) -@dataclass class PrefixDerivation(ElementDerivation): - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.PrefixDerivation - class_class_curie: ClassVar[str] = "linkmltr:PrefixDerivation" - class_name: ClassVar[str] = "PrefixDerivation" - class_model_uri: ClassVar[URIRef] = LINKMLTR.PrefixDerivation - - name: Union[str, PrefixDerivationName] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.name): - self.MissingRequiredField("name") - if not isinstance(self.name, PrefixDerivationName): - self.name = PrefixDerivationName(self.name) - - super().__post_init__(**kwargs) + name: str = Field(None, description="""Name of the element in the target schema""") + copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) + overrides: Optional[Any] = Field( + None, description="""overrides source schema slots""" + ) + is_a: Optional[str] = Field(None) + mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) + value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table that is applied directly to mappings, in order of precedence""", + ) + expression_to_value_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys are expressions""", + ) + expression_to_expression_mappings: Optional[Dict[str, KeyVal]] = Field( + default_factory=dict, + description="""A mapping table in which the keys and values are expressions""", + ) -@dataclass -class Inverse(YAMLRoot): +class Inverse(ConfiguredBaseModel): """ Used for back references """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.Inverse - class_class_curie: ClassVar[str] = "linkmltr:Inverse" - class_name: ClassVar[str] = "Inverse" - class_model_uri: ClassVar[URIRef] = LINKMLTR.Inverse - slot_name: Optional[str] = None - class_name: Optional[str] = None + slot_name: Optional[str] = Field(None) + class_name: Optional[str] = Field(None) - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self.slot_name is not None and not isinstance(self.slot_name, str): - self.slot_name = str(self.slot_name) - - if self.class_name is not None and not isinstance(self.class_name, str): - self.class_name = str(self.class_name) - - super().__post_init__(**kwargs) - - -@dataclass -class KeyVal(YAMLRoot): - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.KeyVal - class_class_curie: ClassVar[str] = "linkmltr:KeyVal" - class_name: ClassVar[str] = "KeyVal" - class_model_uri: ClassVar[URIRef] = LINKMLTR.KeyVal - - key: Union[str, KeyValKey] = None - value: Optional[Union[dict, Any]] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.key): - self.MissingRequiredField("key") - if not isinstance(self.key, KeyValKey): - self.key = KeyValKey(self.key) - - super().__post_init__(**kwargs) - - -@dataclass -class CopyDirective(YAMLRoot): - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = LINKMLTR.CopyDirective - class_class_curie: ClassVar[str] = "linkmltr:CopyDirective" - class_name: ClassVar[str] = "CopyDirective" - class_model_uri: ClassVar[URIRef] = LINKMLTR.CopyDirective - - element_name: Union[str, CopyDirectiveElementName] = None - copy_all: Optional[Union[bool, Bool]] = None - exclude_all: Optional[Union[bool, Bool]] = None - exclude: Optional[Union[dict, Any]] = None - include: Optional[Union[dict, Any]] = None - add: Optional[Union[dict, Any]] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.element_name): - self.MissingRequiredField("element_name") - if not isinstance(self.element_name, CopyDirectiveElementName): - self.element_name = CopyDirectiveElementName(self.element_name) - - if self.copy_all is not None and not isinstance(self.copy_all, Bool): - self.copy_all = Bool(self.copy_all) - - if self.exclude_all is not None and not isinstance(self.exclude_all, Bool): - self.exclude_all = Bool(self.exclude_all) - - super().__post_init__(**kwargs) - - -# Enumerations -class CollectionType(EnumDefinitionImpl): - - SingleValued = PermissibleValue(text="SingleValued") - MultiValued = PermissibleValue(text="MultiValued") - MultiValuedList = PermissibleValue(text="MultiValuedList") - MultiValuedDict = PermissibleValue(text="MultiValuedDict") - - _defn = EnumDefinition( - name="CollectionType", - ) - -# Slots -class slots: - pass -slots.transformationSpecification__id = Slot(uri=LINKMLTR.id, name="transformationSpecification__id", curie=LINKMLTR.curie('id'), - model_uri=LINKMLTR.transformationSpecification__id, domain=None, range=URIRef) +class KeyVal(ConfiguredBaseModel): + key: str = Field(None) + value: Optional[Any] = Field(None) -slots.transformationSpecification__title = Slot(uri=LINKMLTR.title, name="transformationSpecification__title", curie=LINKMLTR.curie('title'), - model_uri=LINKMLTR.transformationSpecification__title, domain=None, range=Optional[str]) -slots.transformationSpecification__prefixes = Slot(uri=LINKMLTR.prefixes, name="transformationSpecification__prefixes", curie=LINKMLTR.curie('prefixes'), - model_uri=LINKMLTR.transformationSpecification__prefixes, domain=None, range=Optional[str]) - -slots.transformationSpecification__source_schema = Slot(uri=LINKMLTR.source_schema, name="transformationSpecification__source_schema", curie=LINKMLTR.curie('source_schema'), - model_uri=LINKMLTR.transformationSpecification__source_schema, domain=None, range=Optional[str]) - -slots.transformationSpecification__target_schema = Slot(uri=LINKMLTR.target_schema, name="transformationSpecification__target_schema", curie=LINKMLTR.curie('target_schema'), - model_uri=LINKMLTR.transformationSpecification__target_schema, domain=None, range=Optional[str]) - -slots.transformationSpecification__class_derivations = Slot(uri=LINKMLTR.class_derivations, name="transformationSpecification__class_derivations", curie=LINKMLTR.curie('class_derivations'), - model_uri=LINKMLTR.transformationSpecification__class_derivations, domain=None, range=Optional[Union[Dict[Union[str, ClassDerivationName], Union[dict, ClassDerivation]], List[Union[dict, ClassDerivation]]]]) - -slots.transformationSpecification__enum_derivations = Slot(uri=LINKMLTR.enum_derivations, name="transformationSpecification__enum_derivations", curie=LINKMLTR.curie('enum_derivations'), - model_uri=LINKMLTR.transformationSpecification__enum_derivations, domain=None, range=Optional[Union[Dict[Union[str, EnumDerivationName], Union[dict, EnumDerivation]], List[Union[dict, EnumDerivation]]]]) - -slots.transformationSpecification__slot_derivations = Slot(uri=LINKMLTR.slot_derivations, name="transformationSpecification__slot_derivations", curie=LINKMLTR.curie('slot_derivations'), - model_uri=LINKMLTR.transformationSpecification__slot_derivations, domain=None, range=Optional[Union[Dict[Union[str, SlotDerivationName], Union[dict, SlotDerivation]], List[Union[dict, SlotDerivation]]]]) - -slots.elementDerivation__name = Slot(uri=LINKMLTR.name, name="elementDerivation__name", curie=LINKMLTR.curie('name'), - model_uri=LINKMLTR.elementDerivation__name, domain=None, range=URIRef) - -slots.elementDerivation__copy_directives = Slot(uri=LINKMLTR.copy_directives, name="elementDerivation__copy_directives", curie=LINKMLTR.curie('copy_directives'), - model_uri=LINKMLTR.elementDerivation__copy_directives, domain=None, range=Optional[Union[Dict[Union[str, CopyDirectiveElementName], Union[dict, CopyDirective]], List[Union[dict, CopyDirective]]]]) - -slots.elementDerivation__is_a = Slot(uri=LINKMLTR.is_a, name="elementDerivation__is_a", curie=LINKMLTR.curie('is_a'), - model_uri=LINKMLTR.elementDerivation__is_a, domain=None, range=Optional[Union[str, ElementDerivationName]]) - -slots.elementDerivation__mixins = Slot(uri=LINKMLTR.mixins, name="elementDerivation__mixins", curie=LINKMLTR.curie('mixins'), - model_uri=LINKMLTR.elementDerivation__mixins, domain=None, range=Optional[Union[Dict[Union[str, ElementDerivationName], Union[dict, ElementDerivation]], List[Union[dict, ElementDerivation]]]]) - -slots.elementDerivation__value_mappings = Slot(uri=LINKMLTR.value_mappings, name="elementDerivation__value_mappings", curie=LINKMLTR.curie('value_mappings'), - model_uri=LINKMLTR.elementDerivation__value_mappings, domain=None, range=Optional[Union[Dict[Union[str, KeyValKey], Union[dict, KeyVal]], List[Union[dict, KeyVal]]]]) - -slots.elementDerivation__expression_to_value_mappings = Slot(uri=LINKMLTR.expression_to_value_mappings, name="elementDerivation__expression_to_value_mappings", curie=LINKMLTR.curie('expression_to_value_mappings'), - model_uri=LINKMLTR.elementDerivation__expression_to_value_mappings, domain=None, range=Optional[Union[Dict[Union[str, KeyValKey], Union[dict, KeyVal]], List[Union[dict, KeyVal]]]]) - -slots.elementDerivation__expression_to_expression_mappings = Slot(uri=LINKMLTR.expression_to_expression_mappings, name="elementDerivation__expression_to_expression_mappings", curie=LINKMLTR.curie('expression_to_expression_mappings'), - model_uri=LINKMLTR.elementDerivation__expression_to_expression_mappings, domain=None, range=Optional[Union[Dict[Union[str, KeyValKey], Union[dict, KeyVal]], List[Union[dict, KeyVal]]]]) - -slots.classDerivation__name = Slot(uri=LINKMLTR.name, name="classDerivation__name", curie=LINKMLTR.curie('name'), - model_uri=LINKMLTR.classDerivation__name, domain=None, range=URIRef) - -slots.classDerivation__populated_from = Slot(uri=LINKMLTR.populated_from, name="classDerivation__populated_from", curie=LINKMLTR.curie('populated_from'), - model_uri=LINKMLTR.classDerivation__populated_from, domain=None, range=Optional[str]) - -slots.classDerivation__joins = Slot(uri=LINKMLTR.joins, name="classDerivation__joins", curie=LINKMLTR.curie('joins'), - model_uri=LINKMLTR.classDerivation__joins, domain=None, range=Optional[Union[Dict[Union[str, AliasedClassAlias], Union[dict, AliasedClass]], List[Union[dict, AliasedClass]]]]) - -slots.classDerivation__slot_derivations = Slot(uri=LINKMLTR.slot_derivations, name="classDerivation__slot_derivations", curie=LINKMLTR.curie('slot_derivations'), - model_uri=LINKMLTR.classDerivation__slot_derivations, domain=None, range=Optional[Union[Dict[Union[str, SlotDerivationName], Union[dict, SlotDerivation]], List[Union[dict, SlotDerivation]]]]) - -slots.aliasedClass__alias = Slot(uri=LINKMLTR.alias, name="aliasedClass__alias", curie=LINKMLTR.curie('alias'), - model_uri=LINKMLTR.aliasedClass__alias, domain=None, range=URIRef) - -slots.aliasedClass__class_named = Slot(uri=LINKMLTR.class_named, name="aliasedClass__class_named", curie=LINKMLTR.curie('class_named'), - model_uri=LINKMLTR.aliasedClass__class_named, domain=None, range=Optional[str]) - -slots.slotDerivation__name = Slot(uri=LINKMLTR.name, name="slotDerivation__name", curie=LINKMLTR.curie('name'), - model_uri=LINKMLTR.slotDerivation__name, domain=None, range=URIRef) - -slots.slotDerivation__populated_from = Slot(uri=LINKMLTR.populated_from, name="slotDerivation__populated_from", curie=LINKMLTR.curie('populated_from'), - model_uri=LINKMLTR.slotDerivation__populated_from, domain=None, range=Optional[str]) - -slots.slotDerivation__expr = Slot(uri=LINKMLTR.expr, name="slotDerivation__expr", curie=LINKMLTR.curie('expr'), - model_uri=LINKMLTR.slotDerivation__expr, domain=None, range=Optional[str]) - -slots.slotDerivation__inverse_of = Slot(uri=LINKMLTR.inverse_of, name="slotDerivation__inverse_of", curie=LINKMLTR.curie('inverse_of'), - model_uri=LINKMLTR.slotDerivation__inverse_of, domain=None, range=Optional[Union[dict, Inverse]]) - -slots.slotDerivation__hide = Slot(uri=LINKMLTR.hide, name="slotDerivation__hide", curie=LINKMLTR.curie('hide'), - model_uri=LINKMLTR.slotDerivation__hide, domain=None, range=Optional[Union[bool, Bool]]) - -slots.slotDerivation__type_designator = Slot(uri=LINKMLTR.type_designator, name="slotDerivation__type_designator", curie=LINKMLTR.curie('type_designator'), - model_uri=LINKMLTR.slotDerivation__type_designator, domain=None, range=Optional[Union[bool, Bool]]) - -slots.slotDerivation__cast_collection_as = Slot(uri=LINKMLTR.cast_collection_as, name="slotDerivation__cast_collection_as", curie=LINKMLTR.curie('cast_collection_as'), - model_uri=LINKMLTR.slotDerivation__cast_collection_as, domain=None, range=Optional[Union[str, "CollectionType"]]) - -slots.enumDerivation__name = Slot(uri=LINKMLTR.name, name="enumDerivation__name", curie=LINKMLTR.curie('name'), - model_uri=LINKMLTR.enumDerivation__name, domain=None, range=URIRef) - -slots.enumDerivation__populated_from = Slot(uri=LINKMLTR.populated_from, name="enumDerivation__populated_from", curie=LINKMLTR.curie('populated_from'), - model_uri=LINKMLTR.enumDerivation__populated_from, domain=None, range=Optional[str]) - -slots.enumDerivation__expr = Slot(uri=LINKMLTR.expr, name="enumDerivation__expr", curie=LINKMLTR.curie('expr'), - model_uri=LINKMLTR.enumDerivation__expr, domain=None, range=Optional[str]) - -slots.enumDerivation__hide = Slot(uri=LINKMLTR.hide, name="enumDerivation__hide", curie=LINKMLTR.curie('hide'), - model_uri=LINKMLTR.enumDerivation__hide, domain=None, range=Optional[Union[bool, Bool]]) - -slots.enumDerivation__permissible_value_derivations = Slot(uri=LINKMLTR.permissible_value_derivations, name="enumDerivation__permissible_value_derivations", curie=LINKMLTR.curie('permissible_value_derivations'), - model_uri=LINKMLTR.enumDerivation__permissible_value_derivations, domain=None, range=Optional[Union[Dict[Union[str, PermissibleValueDerivationName], Union[dict, PermissibleValueDerivation]], List[Union[dict, PermissibleValueDerivation]]]]) - -slots.permissibleValueDerivation__name = Slot(uri=LINKMLTR.name, name="permissibleValueDerivation__name", curie=LINKMLTR.curie('name'), - model_uri=LINKMLTR.permissibleValueDerivation__name, domain=None, range=URIRef) - -slots.permissibleValueDerivation__expr = Slot(uri=LINKMLTR.expr, name="permissibleValueDerivation__expr", curie=LINKMLTR.curie('expr'), - model_uri=LINKMLTR.permissibleValueDerivation__expr, domain=None, range=Optional[str]) - -slots.permissibleValueDerivation__populated_from = Slot(uri=LINKMLTR.populated_from, name="permissibleValueDerivation__populated_from", curie=LINKMLTR.curie('populated_from'), - model_uri=LINKMLTR.permissibleValueDerivation__populated_from, domain=None, range=Optional[str]) - -slots.permissibleValueDerivation__hide = Slot(uri=LINKMLTR.hide, name="permissibleValueDerivation__hide", curie=LINKMLTR.curie('hide'), - model_uri=LINKMLTR.permissibleValueDerivation__hide, domain=None, range=Optional[Union[bool, Bool]]) - -slots.inverse__slot_name = Slot(uri=LINKMLTR.slot_name, name="inverse__slot_name", curie=LINKMLTR.curie('slot_name'), - model_uri=LINKMLTR.inverse__slot_name, domain=None, range=Optional[str]) - -slots.inverse__class_name = Slot(uri=LINKMLTR.class_name, name="inverse__class_name", curie=LINKMLTR.curie('class_name'), - model_uri=LINKMLTR.inverse__class_name, domain=None, range=Optional[str]) - -slots.keyVal__key = Slot(uri=LINKMLTR.key, name="keyVal__key", curie=LINKMLTR.curie('key'), - model_uri=LINKMLTR.keyVal__key, domain=None, range=URIRef) - -slots.keyVal__value = Slot(uri=LINKMLTR.value, name="keyVal__value", curie=LINKMLTR.curie('value'), - model_uri=LINKMLTR.keyVal__value, domain=None, range=Optional[Union[dict, Any]]) - -slots.copyDirective__element_name = Slot(uri=LINKMLTR.element_name, name="copyDirective__element_name", curie=LINKMLTR.curie('element_name'), - model_uri=LINKMLTR.copyDirective__element_name, domain=None, range=URIRef) - -slots.copyDirective__copy_all = Slot(uri=LINKMLTR.copy_all, name="copyDirective__copy_all", curie=LINKMLTR.curie('copy_all'), - model_uri=LINKMLTR.copyDirective__copy_all, domain=None, range=Optional[Union[bool, Bool]]) - -slots.copyDirective__exclude_all = Slot(uri=LINKMLTR.exclude_all, name="copyDirective__exclude_all", curie=LINKMLTR.curie('exclude_all'), - model_uri=LINKMLTR.copyDirective__exclude_all, domain=None, range=Optional[Union[bool, Bool]]) - -slots.copyDirective__exclude = Slot(uri=LINKMLTR.exclude, name="copyDirective__exclude", curie=LINKMLTR.curie('exclude'), - model_uri=LINKMLTR.copyDirective__exclude, domain=None, range=Optional[Union[dict, Any]]) - -slots.copyDirective__include = Slot(uri=LINKMLTR.include, name="copyDirective__include", curie=LINKMLTR.curie('include'), - model_uri=LINKMLTR.copyDirective__include, domain=None, range=Optional[Union[dict, Any]]) +class CopyDirective(ConfiguredBaseModel): + """ + Instructs a Schema Mapper in how to map to a target schema. Not used for data transformation. + """ -slots.copyDirective__add = Slot(uri=LINKMLTR.add, name="copyDirective__add", curie=LINKMLTR.curie('add'), - model_uri=LINKMLTR.copyDirective__add, domain=None, range=Optional[Union[dict, Any]]) \ No newline at end of file + element_name: str = Field(None) + copy_all: Optional[bool] = Field(None) + exclude_all: Optional[bool] = Field(None) + exclude: Optional[Any] = Field(None) + include: Optional[Any] = Field(None) + add: Optional[Any] = Field(None) + + +# Update forward refs +# see https://pydantic-docs.helpmanual.io/usage/postponed_annotations/ +TransformationSpecification.update_forward_refs() +ElementDerivation.update_forward_refs() +ClassDerivation.update_forward_refs() +AliasedClass.update_forward_refs() +SlotDerivation.update_forward_refs() +EnumDerivation.update_forward_refs() +PermissibleValueDerivation.update_forward_refs() +PrefixDerivation.update_forward_refs() +Inverse.update_forward_refs() +KeyVal.update_forward_refs() +CopyDirective.update_forward_refs() diff --git a/src/linkml_transformer/datamodel/transformer_model.yaml b/src/linkml_transformer/datamodel/transformer_model.yaml index 7db0e57..5f13e3a 100644 --- a/src/linkml_transformer/datamodel/transformer_model.yaml +++ b/src/linkml_transformer/datamodel/transformer_model.yaml @@ -40,6 +40,9 @@ classes: description: human readable title for this transformation specification prefixes: description: maps prefixes to URL expansions + range: KeyVal + multivalued: true + inlined: true source_schema: description: name of the schema that describes the source (input) objects target_schema: @@ -77,6 +80,11 @@ classes: range: CopyDirective multivalued: true inlined: true + overrides: + description: overrides source schema slots + range: Any + #multivalued: true + #inlined: true is_a: range: ElementDerivation mixins: @@ -108,9 +116,9 @@ classes: description: >- A specification of how to derive a target class from a source class. attributes: - name: - key: true - description: Name of the class in the target schema + #name: + # key: true + # description: Name of the class in the target schema populated_from: range: string description: Name of the class in the source schema @@ -150,6 +158,9 @@ classes: description: >- An expression to be evaluated on the source object to derive the target slot. Should be specified using the LinkML expression language. + range: + slot_uri: linkml:range + range: string inverse_of: range: Inverse description: >- @@ -164,6 +175,8 @@ classes: range: boolean cast_collection_as: range: CollectionType + #range: + # slot_uri: linkml:range EnumDerivation: is_a: ElementDerivation @@ -227,6 +240,8 @@ classes: range: Any CopyDirective: + description: >- + Instructs a Schema Mapper in how to map to a target schema. Not used for data transformation. attributes: element_name: key: true diff --git a/src/linkml_transformer/schema_mapper/schema_mapper.py b/src/linkml_transformer/schema_mapper/schema_mapper.py index 8e36327..d3e926b 100644 --- a/src/linkml_transformer/schema_mapper/schema_mapper.py +++ b/src/linkml_transformer/schema_mapper/schema_mapper.py @@ -1,9 +1,13 @@ +import logging +from collections import defaultdict from copy import copy -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Dict, List, Optional from linkml_runtime import SchemaView -from linkml_runtime.linkml_model import (ClassDefinition, Element, - SchemaDefinition, SlotDefinition) +from linkml_runtime.linkml_model import (ClassDefinition, ClassDefinitionName, + Element, SchemaDefinition, + SlotDefinition) from linkml_transformer.datamodel.transformer_model import ( ClassDerivation, CopyDirective, TransformationSpecification) @@ -17,11 +21,15 @@ class SchemaMapper: source_schemaview: SchemaView = None + source_to_target_class_mappings: Dict[str, List[str]] = field( + default_factory=lambda: defaultdict(list) + ) + def derive_schema( self, specification: TransformationSpecification ) -> SchemaDefinition: """ - Compile a transformation specification into a schema. + Use a transformation specification to generate a target/profile schema from a source schema. :param specification: :return: @@ -32,6 +40,8 @@ def derive_schema( for class_derivation in specification.class_derivations.values(): class_definition = self._derive_class(class_derivation) target_schema.classes[class_definition.name] = class_definition + for cd in target_schema.classes.values(): + self._rewire_class(cd) return target_schema def _derive_class(self, class_derivation: ClassDerivation) -> ClassDefinition: @@ -43,6 +53,7 @@ def _derive_class(self, class_derivation: ClassDerivation) -> ClassDefinition: populated_from = class_derivation.name source_class = self.source_schemaview.get_class(populated_from) if source_class is None: + logging.warning(f"No such class {populated_from}") target_class = ClassDefinition(name=class_derivation.name) else: target_class = copy(source_class) @@ -53,6 +64,7 @@ def _derive_class(self, class_derivation: ClassDerivation) -> ClassDefinition: for slot_derivation in class_derivation.slot_derivations.values(): slot_definition = self._derive_slot(slot_derivation) target_class.attributes[slot_definition.name] = slot_definition + self.source_to_target_class_mappings[populated_from].append(target_class.name) return target_class def _derive_slot(self, slot_derivation) -> SlotDefinition: @@ -70,6 +82,32 @@ def _derive_slot(self, slot_derivation) -> SlotDefinition: target_slot.name = slot_derivation.name return target_slot + def _rewire_class(self, class_definition: ClassDefinition): + if class_definition.is_a: + class_definition.is_a = self._rewire_parent( + class_definition, class_definition.is_a + ) + mixins = [ + self._rewire_parent(class_definition, m) for m in class_definition.mixins + ] + class_definition.mixins = [m for m in mixins if m is not None] + + def _rewire_parent( + self, class_definition: ClassDefinition, parent: ClassDefinitionName + ) -> Optional[str]: + if parent in self.source_to_target_class_mappings: + new_parents = self.source_to_target_class_mappings[parent] + if len(new_parents) > 1: + raise ValueError( + f"Cannot rewire to non-isomorphic mappings {parent} => {new_parents}" + ) + if len(new_parents) == 1: + return new_parents[0] + parent_cls = self.source_schemaview.get_class(parent) + if parent_cls.is_a: + return self._rewire_parent(class_definition, parent_cls.is_a) + return None + def copy_attributes( self, target_element: Element, diff --git a/src/linkml_transformer/transformer/inference.py b/src/linkml_transformer/transformer/inference.py index 4f4c351..0464dec 100644 --- a/src/linkml_transformer/transformer/inference.py +++ b/src/linkml_transformer/transformer/inference.py @@ -2,10 +2,13 @@ from linkml_runtime import SchemaView -from linkml_transformer.datamodel.transformer_model import TransformationSpecification, SlotDerivation +from linkml_transformer.datamodel.transformer_model import ( + SlotDerivation, TransformationSpecification) -def induce_missing_values(specification: TransformationSpecification, source_schemaview: SchemaView): +def induce_missing_values( + specification: TransformationSpecification, source_schemaview: SchemaView +): """ Infer missing values in a specification. @@ -22,7 +25,7 @@ def induce_missing_values(specification: TransformationSpecification, source_sch for sn in source_schemaview.class_slots(src_cls.name): if sn in cd.slot_derivations: continue - #if slot_match == "*" or re.match(slot_match, sn): + # if slot_match == "*" or re.match(slot_match, sn): # sd = SlotDerivation(name=sn, populated_from=sn) # print(f"Adding {src_cls_name} . {sd}") # cd.slot_derivations[sd.name] = sd diff --git a/src/linkml_transformer/transformer/object_transformer.py b/src/linkml_transformer/transformer/object_transformer.py index 654103e..0b0e4d6 100644 --- a/src/linkml_transformer/transformer/object_transformer.py +++ b/src/linkml_transformer/transformer/object_transformer.py @@ -135,16 +135,28 @@ def transform( source_class_slot_range = source_class_slot.range if source_class_slot.multivalued: if isinstance(v, list): - v = [self.transform(v1, source_class_slot_range, target_range) for v1 in v] + v = [ + self.transform(v1, source_class_slot_range, target_range) + for v1 in v + ] elif isinstance(v, dict): - v = [self.transform(v1, source_class_slot_range, target_range) for v1 in v] + v = [ + self.transform(v1, source_class_slot_range, target_range) + for v1 in v + ] else: v = [v] else: v = self.transform(v, source_class_slot_range, target_range) - if self._coerce_to_multivalued(slot_derivation, class_deriv) and v is not None and not isinstance(v, list): + if ( + self._coerce_to_multivalued(slot_derivation, class_deriv) + and v is not None + and not isinstance(v, list) + ): v = [v] - if self._coerce_to_singlevalued(slot_derivation, class_deriv) and isinstance(v, list): + if self._coerce_to_singlevalued( + slot_derivation, class_deriv + ) and isinstance(v, list): if len(v) > 1: raise ValueError(f"Cannot coerce multiple values {v}") if len(v) == 0: @@ -169,4 +181,3 @@ def transform_object( # raise ValueError(f"Do not know how to handle type: {typ}") tr_obj_dict = self.transform(source_obj, typ_name) return target_class(**tr_obj_dict) - diff --git a/src/linkml_transformer/transformer/transformer.py b/src/linkml_transformer/transformer/transformer.py index fd027ac..4b162b0 100644 --- a/src/linkml_transformer/transformer/transformer.py +++ b/src/linkml_transformer/transformer/transformer.py @@ -7,16 +7,17 @@ from typing import Any, Dict, Optional, Type, Union import yaml +from curies import Converter from linkml_runtime import SchemaView from linkml_runtime.loaders import yaml_loader from linkml_runtime.processing.referencevalidator import ReferenceValidator from linkml_runtime.utils.introspection import package_schemaview from linkml_runtime.utils.yamlutils import YAMLRoot from pydantic import BaseModel -from curies import Converter from linkml_transformer.datamodel.transformer_model import ( - ClassDerivation, TransformationSpecification, SlotDerivation, CollectionType) + ClassDerivation, CollectionType, SlotDerivation, + TransformationSpecification) from linkml_transformer.transformer.inference import induce_missing_values logger = logging.getLogger(__name__) @@ -87,7 +88,9 @@ def load_transformer_specification(self, path: Union[str, Path]): # self.specification = yaml_loader.load(str(path), TransformationSpecification) with open(path) as f: obj = yaml.safe_load(f) - normalizer = ReferenceValidator(package_schemaview("linkml_transformer.datamodel.transformer_model")) + normalizer = ReferenceValidator( + package_schemaview("linkml_transformer.datamodel.transformer_model") + ) normalizer.expand_all = True obj = normalizer.normalize(obj) self.specification = TransformationSpecification(**obj) @@ -101,7 +104,6 @@ def derived_specification(self) -> Optional[TransformationSpecification]: induce_missing_values(self._derived_specification, self.source_schemaview) return self._derived_specification - def _get_class_derivation(self, target_class_name) -> ClassDerivation: spec = self.derived_specification matching_tgt_class_derivs = [ @@ -116,7 +118,9 @@ def _get_class_derivation(self, target_class_name) -> ClassDerivation: ) return matching_tgt_class_derivs[0] - def _coerce_to_multivalued(self, slot_derivation: SlotDerivation, class_derivation: ClassDerivation): + def _coerce_to_multivalued( + self, slot_derivation: SlotDerivation, class_derivation: ClassDerivation + ): cast_as = slot_derivation.cast_collection_as if cast_as and cast_as == CollectionType.MultiValued: return True @@ -127,7 +131,9 @@ def _coerce_to_multivalued(self, slot_derivation: SlotDerivation, class_derivati return True return False - def _coerce_to_singlevalued(self, slot_derivation: SlotDerivation, class_derivation: ClassDerivation): + def _coerce_to_singlevalued( + self, slot_derivation: SlotDerivation, class_derivation: ClassDerivation + ): cast_as = slot_derivation.cast_collection_as if cast_as and cast_as == CollectionType(CollectionType.SingleValued): return True @@ -143,7 +149,9 @@ def curie_converter(self) -> Converter: if not self._curie_converter: self._curie_converter = Converter([]) for prefix in self.source_schemaview.schema.prefixes.values(): - self._curie_converter.add_prefix(prefix.prefix_prefix, prefix.prefix_reference) + self._curie_converter.add_prefix( + prefix.prefix_prefix, prefix.prefix_reference + ) for prefix in self.specification.prefixes.values(): self._curie_converter.add_prefix(prefix.key, prefix.value) return self._curie_converter @@ -153,5 +161,3 @@ def expand_curie(self, curie: str) -> str: def compress_uri(self, uri: str) -> str: return self.curie_converter.compress(uri) - - diff --git a/src/linkml_transformer/utils/inverter.py b/src/linkml_transformer/utils/inverter.py index c6fa7b5..060775e 100644 --- a/src/linkml_transformer/utils/inverter.py +++ b/src/linkml_transformer/utils/inverter.py @@ -1,5 +1,8 @@ -from linkml_transformer.datamodel.transformer_model import TransformationSpecification +from linkml_transformer.datamodel.transformer_model import \ + TransformationSpecification -def invert_transformation_specification(specification: TransformationSpecification) -> TransformationSpecification: +def invert_transformation_specification( + specification: TransformationSpecification, +) -> TransformationSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/utils/multi_file_transformer.py b/src/linkml_transformer/utils/multi_file_transformer.py index fbf21d7..f53dcc1 100644 --- a/src/linkml_transformer/utils/multi_file_transformer.py +++ b/src/linkml_transformer/utils/multi_file_transformer.py @@ -11,7 +11,7 @@ from io import StringIO from pathlib import Path from types import ModuleType -from typing import Union, Any, Mapping, Optional, List, TextIO +from typing import Any, List, Mapping, Optional, TextIO, Union import click import yaml @@ -21,25 +21,30 @@ from linkml_runtime.utils.formatutils import camelcase from pydantic import BaseModel +from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper from linkml_transformer.transformer.object_transformer import ObjectTransformer class Step(BaseModel): source_data: str target_data: Optional[str] = None + source_class: str = None + target_class: str = None + class Transformation(BaseModel): description: Optional[str] = None - source_schema: str - transformation_specification: str + source_schema: str = None + target_schema: str = None + transformation_specification: str = None steps: List[Step] = None + class Instructions(BaseModel): description: Optional[str] = None transformations: List[Transformation] = [] - @dataclass class MultiFileTransformer: """ @@ -55,14 +60,17 @@ class MultiFileTransformer: """ source_schema_directory_base: str = field(default_factory=lambda: "source") - transform_specification_directory_base: str = field(default_factory=lambda: "transform") - source_data_directory_base: str = field(default_factory=lambda: "data") - target_data_directory_base: str = field(default_factory=lambda: "target") - - input_formats: Optional[List[str]] = field(default_factory=lambda: ['yaml']) + transform_specification_directory_base: str = field( + default_factory=lambda: "transform" + ) + source_data_directory_base: str = field(default_factory=lambda: "data") + target_schema_directory_base: str = field(default_factory=lambda: "target") + target_data_directory_base: str = field(default_factory=lambda: "output") + + input_formats: Optional[List[str]] = field(default_factory=lambda: ["yaml"]) """Expected formats for input data""" - output_formats: Optional[List[str]] = field(default_factory=lambda: ['yaml']) + output_formats: Optional[List[str]] = field(default_factory=lambda: ["yaml"]) prefix_map: Optional[Mapping[str, str]] = None """Custom prefix map, for emitting RDF/turtle.""" @@ -92,49 +100,97 @@ def infer_instructions(self, root_directory: Union[str, Path]) -> Instructions: source_schema_directory = root_directory / self.source_schema_directory_base if not source_schema_directory.exists(): raise ValueError(f"Expected {source_schema_directory}") - transform_specification_directory = root_directory / self.transform_specification_directory_base + transform_specification_directory = ( + root_directory / self.transform_specification_directory_base + ) if not transform_specification_directory.exists(): raise ValueError(f"Expected {transform_specification_directory}") + target_schema_directory = root_directory / self.target_schema_directory_base input_schemas = glob.glob(os.path.join(str(source_schema_directory), "*.yaml")) if not input_schemas: raise ValueError(f"Expected schemas in {source_schema_directory}") + transform_files = glob.glob( + os.path.join(str(transform_specification_directory), "*.transform.yaml") + ) instructions = Instructions(description="auto-inducted") for input_schema in input_schemas: - transform_files = glob.glob(os.path.join(str(transform_specification_directory), "*.transform.yaml")) for transform_file in transform_files: - tr = Transformation(source_schema=input_schema, - transform_specification=transform_file) + tr = Transformation( + source_schema=input_schema, + transformation_specification=transform_file, + steps=[], + ) target_schema_base = None if len(input_schemas) != 1: # resolve which schema the transform applies to - matches = re.match(r'^(\w+)-to-(\w+)\.', transform_file) + matches = re.match(r"^(\w+)-to-(\w+)\.", transform_file) if not matches: raise ValueError(f"Ambiguous: {transform_file}") src, target_schema_base = matches.group(1, 2) if src not in input_schema: + print(f"SKIP {input_schema}") continue instructions.transformations.append(tr) - data_files = glob.glob(os.path.join(str(self.source_data_directory_base), "*.yaml")) + data_files = glob.glob( + os.path.join( + str(root_directory / self.source_data_directory_base), "*.yaml" + ) + ) for data_file in data_files: if target_schema_base and target_schema_base not in data_file: continue - step = Step(source_data=data_file) + target_data = str( + Path(self.target_data_directory_base) / Path(data_file).name + ) + target_data = target_data.replace(".yaml", ".transformed.yaml") + step = Step(source_data=data_file, target_data=target_data) + step.source_class = Path(data_file).stem.split("-")[-2] tr.steps.append(step) + target_schemas = glob.glob( + os.path.join(str(target_schema_directory), "*.yaml") + ) + if len(target_schemas) > 1: + target_schemas = [ + s for s in target_schemas if target_schema_base in s + ] + if len(target_schemas) != 1: + raise ValueError( + f"Could not determine target schema from: {target_schemas}" + ) + if target_schemas: + tr.target_schema = target_schemas[0] + else: + tr.target_schema = str( + Path(target_schema_directory) / "target.yaml" + ) if not tr.steps: raise ValueError(f"Could not infer steps from {data_files}") + if not tr.transformation_specification: + raise ValueError(f"No spec {tr}") return instructions - def process_instructions(self, instructions: Instructions, root_directory: Optional[Union[str, Path]], test_mode=False): + def process_instructions( + self, + instructions: Instructions, + root_directory: Optional[Union[str, Path]], + output_directory=None, + test_mode=False, + ): """ Process a set of instructions to transform one or more files. :param instructions: :param root_directory: + :param output_directory: :param test_mode: :return: """ if isinstance(root_directory, str): - directory = Path(root_directory) + root_directory = Path(root_directory) + if isinstance(output_directory, str): + root_directory = Path(output_directory) + if not output_directory: + output_directory = root_directory logging.info(f"Processing: {instructions.description}") for tr in instructions.transformations: source_schema_path = str(root_directory / tr.source_schema) @@ -142,11 +198,36 @@ def process_instructions(self, instructions: Instructions, root_directory: Optio transformer = ObjectTransformer() transformer.load_source_schema(source_schema_path) tr_path = str(root_directory / tr.transformation_specification) - print(tr_path) transformer.load_transformer_specification(tr_path) + if tr.target_schema: + target_schema_path = root_directory / tr.target_schema + mapper = SchemaMapper() + mapper.source_schemaview = sv + target_schema_obj = mapper.derive_schema(transformer.specification) + if not target_schema_path.exists(): + target_schema_path.parent.mkdir(exist_ok=True, parents=True) + yaml_dumper.dump(target_schema_obj, str(target_schema_path)) for step in tr.steps: input_obj = yaml.safe_load(open(str(root_directory / step.source_data))) - print(input_obj) - target_obj = transformer.transform(input_obj) - print(target_obj) - + transformer.index(input_obj, step.source_class) + target_obj = transformer.transform( + input_obj, source_type=step.source_class + ) + if step.target_data: + out_path = output_directory / step.target_data + out_path.parent.mkdir(parents=True, exist_ok=True) + if out_path.exists(): + line1 = open(str(out_path)).readline() + overwrite = "OVERWRITE" in line1 + with open(str(out_path)) as f: + compare_obj = yaml.safe_load(f) + if compare_obj != target_obj: + if test_mode and not overwrite: + raise ValueError( + f"Output different than expected: {compare_obj}" + ) + else: + logging.warning(f"Different: {compare_obj}") + with open(str(out_path), "w", encoding="utf-8") as f: + yaml_str = yaml.dump(target_obj, sort_keys=False) + f.write(yaml_str) diff --git a/tests/input/examples/README.md b/tests/input/examples/README.md new file mode 100644 index 0000000..4296458 --- /dev/null +++ b/tests/input/examples/README.md @@ -0,0 +1,31 @@ +# Transformer Examples + +This folder contains examples that serve as both documentation and unit tests. + +Each subfolder contains independent examples, that contain at least: + + - 1 source schema + - 1 transformation + - 1 test input data file (conforming to source schema) + +when the test suite is executed, it will include a derived schema for each source schema, plus one output file +for every input file. If these files exist, then the contents of a fresh derivation is checked against them. + +The folder layout is: + + - `foobar` + - `source` + - `foo.yaml` ## source schema + - `transform` + - `foo-to-bar.transform.yaml` + - `target` + - `bar.yaml` ## target schema, autogenerated if not present + - `data` + - `Foo-001.yaml` ## Example file 1, instantiating Foo class for foo schema + - `Foo-002.yaml` ## Example file 2, instantiating Foo class for foo schema + - ... + - `output` ## derived files, conforming to bar, autogenerated if not present + - `Bar-001.yaml` + - `Bar-002.yaml` + +Optionally, an `instructions.yaml` file may be present. If not the structure of the folder is inferred. \ No newline at end of file diff --git a/tests/input/examples/biolink/data/gene-001.yaml b/tests/input/examples/biolink/data/gene-001.yaml new file mode 100644 index 0000000..adbcc82 --- /dev/null +++ b/tests/input/examples/biolink/data/gene-001.yaml @@ -0,0 +1,3 @@ +id: HGNC:1 +symbol: foo +name: foo bar \ No newline at end of file diff --git a/tests/input/examples/biolink/source/biolink-model.yaml b/tests/input/examples/biolink/source/biolink-model.yaml index 740f3fb..fd31e41 100644 --- a/tests/input/examples/biolink/source/biolink-model.yaml +++ b/tests/input/examples/biolink/source/biolink-model.yaml @@ -5476,7 +5476,6 @@ classes: abstract: true slots: - predicate mappings - tree_root: true predicate mapping: description: >- diff --git a/tests/input/examples/biolink/transform/biolink-example-profile.transform.yaml b/tests/input/examples/biolink/transform/biolink-example-profile.transform.yaml index 614d316..7367c33 100644 --- a/tests/input/examples/biolink/transform/biolink-example-profile.transform.yaml +++ b/tests/input/examples/biolink/transform/biolink-example-profile.transform.yaml @@ -5,32 +5,38 @@ prefixes: source_schema: biolink target_schema: monarch -slot_names_match: true # autopopulates slot.populated_from -class_names_match: true # autopopulates class.populated_from +#slot_names_match: true # autopopulates slot.populated_from +#class_names_match: true # autopopulates class.populated_from class_derivations: NamedThing: + populated_from: named thing slot_derivations: id: name: overrides: category: Gene: + populated_from: gene slot_derivations: symbol: overrides: #is_a: NamedThing # would be autopopulated Disease: + populated_from: disease overrides: id_prefixes: - MONDO PhenotypicFeature: + populated_from: phenotypic feature Association: + populated_from: association slot_derivations: subject: predicate: object: GeneToPhenotypicFeatureAssociation: + populated_from: gene to phenotypic feature associations overrides: slot_usage: predicate: diff --git a/tests/input/examples/measurements/instructions.yaml b/tests/input/examples/measurements/instructions.yaml index c839a65..6c78af6 100644 --- a/tests/input/examples/measurements/instructions.yaml +++ b/tests/input/examples/measurements/instructions.yaml @@ -1,6 +1,10 @@ description: Examples of transforming measurements transformations: - source_schema: source/quantity_value.yaml + target_schema: target/quantity_value_flat.yaml transformation_specification: transform/qv-to-scalar.transform.yaml steps: - source_data: data/PersonQuantityValue-001.yaml + target_data: output/PersonQuantityValue-001.transformed.yaml + - source_data: data/PersonQuantityValue-002.yaml + target_data: output/PersonQuantityValue-002.transformed.yaml diff --git a/tests/input/examples/personinfo_basic/model/agent_model.py b/tests/input/examples/personinfo_basic/model/agent_model.py index ceb17fa..7282e05 100644 --- a/tests/input/examples/personinfo_basic/model/agent_model.py +++ b/tests/input/examples/personinfo_basic/model/agent_model.py @@ -7,34 +7,26 @@ # license: https://creativecommons.org/publicdomain/zero/1.0/ import dataclasses -import sys import re -from jsonasobj2 import JsonObj, as_dict -from typing import Optional, List, Union, Dict, ClassVar, Any +import sys from dataclasses import dataclass -from linkml_runtime.linkml_model.meta import ( - EnumDefinition, - PermissibleValue, - PvFormulaOptions, -) +from typing import Any, ClassVar, Dict, List, Optional, Union -from linkml_runtime.utils.slot import Slot -from linkml_runtime.utils.metamodelcore import empty_list, empty_dict, bnode -from linkml_runtime.utils.yamlutils import ( - YAMLRoot, - extended_str, - extended_float, - extended_int, -) -from linkml_runtime.utils.dataclass_extensions_376 import ( - dataclasses_init_fn_with_kwargs, -) -from linkml_runtime.utils.formatutils import camelcase, underscore, sfx +from jsonasobj2 import JsonObj, as_dict +from linkml_runtime.linkml_model.meta import (EnumDefinition, PermissibleValue, + PvFormulaOptions) +from linkml_runtime.linkml_model.types import Boolean, Date, Float, String +from linkml_runtime.utils.curienamespace import CurieNamespace +from linkml_runtime.utils.dataclass_extensions_376 import \ + dataclasses_init_fn_with_kwargs from linkml_runtime.utils.enumerations import EnumDefinitionImpl +from linkml_runtime.utils.formatutils import camelcase, sfx, underscore +from linkml_runtime.utils.metamodelcore import (Bool, XSDDate, bnode, + empty_dict, empty_list) +from linkml_runtime.utils.slot import Slot +from linkml_runtime.utils.yamlutils import (YAMLRoot, extended_float, + extended_int, extended_str) from rdflib import Namespace, URIRef -from linkml_runtime.utils.curienamespace import CurieNamespace -from linkml_runtime.linkml_model.types import Boolean, Date, Float, String -from linkml_runtime.utils.metamodelcore import Bool, XSDDate metamodel_version = "1.7.0" version = None diff --git a/tests/test_transformer/test_object_transformer.py b/tests/test_transformer/test_object_transformer.py index b2c26dc..49d10dc 100644 --- a/tests/test_transformer/test_object_transformer.py +++ b/tests/test_transformer/test_object_transformer.py @@ -3,12 +3,8 @@ import yaml from linkml_runtime import SchemaView -from linkml_runtime.dumpers import yaml_dumper -from linkml_runtime.linkml_model import ( - ClassDefinition, - SlotDefinition, - SchemaDefinition, -) +from linkml_runtime.linkml_model import (ClassDefinition, SchemaDefinition, + SlotDefinition) from linkml_runtime.loaders import yaml_loader import tests.input.examples.flattening.model.denormalized_model as sssom_tgt_dm @@ -18,16 +14,9 @@ from linkml_transformer.datamodel.transformer_model import * from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.utils.dynamic_object import dynamic_object -from tests import ( - DENORM_SCHEMA, - DENORM_SPECIFICATION, - FLATTENING_DATA, - NORM_SCHEMA, - PERSONINFO_DATA, - PERSONINFO_SRC_SCHEMA, - PERSONINFO_TGT_SCHEMA, - PERSONINFO_TR, -) +from tests import (DENORM_SCHEMA, DENORM_SPECIFICATION, FLATTENING_DATA, + NORM_SCHEMA, PERSONINFO_DATA, PERSONINFO_SRC_SCHEMA, + PERSONINFO_TGT_SCHEMA, PERSONINFO_TR) AGE_STRING = "33 years" diff --git a/tests/test_transformer/test_transformer_examples.py b/tests/test_transformer/test_transformer_examples.py index 5ab17f0..6028acc 100644 --- a/tests/test_transformer/test_transformer_examples.py +++ b/tests/test_transformer/test_transformer_examples.py @@ -4,11 +4,18 @@ from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.utils.dynamic_object import dynamic_object -from linkml_transformer.utils.multi_file_transformer import MultiFileTransformer +from linkml_transformer.utils.multi_file_transformer import \ + MultiFileTransformer from tests import EXAMPLE_DIR EXAMPLES = [ - ("measurements", "quantity_value", "qv-to-scalar", "PersonQuantityValue-001", 172.0), + ( + "measurements", + "quantity_value", + "qv-to-scalar", + "PersonQuantityValue-001", + 172.0, + ), ("measurements", "quantity_value", "qv-to-scalar", "PersonQuantityValue-002", None), ] @@ -50,11 +57,27 @@ def test_all(self): :return: """ mft = MultiFileTransformer() - dirs = ["measurements"] + dirs = [ + "measurements", + "flattening", + "personinfo_basic", + "type_coercion", + "biolink", + ] for dir in dirs: full_dir = EXAMPLE_DIR / dir instructions = mft.infer_instructions(full_dir) - print(instructions) - obj = mft.process_instructions(instructions, full_dir) - print(obj) + # print(yaml.dump(instructions.dict())) + obj = mft.process_instructions(instructions, full_dir, test_mode=True) + def test_regenerate(self): + """ + Use this to regenerate test examples + """ + mft = MultiFileTransformer() + dirs = ["biolink"] + for dir in dirs: + full_dir = EXAMPLE_DIR / dir + instructions = mft.infer_instructions(full_dir) + # print(yaml.dump(instructions.dict())) + obj = mft.process_instructions(instructions, full_dir, test_mode=False) From 8a550916fdfff23c547b24eb6f80e243455e198c Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 2 May 2023 01:49:10 +0200 Subject: [PATCH 04/15] adding tests. adding skeleton python compiler --- src/linkml_transformer/cli/cli.py | 15 ++-- .../compiler/awk_compiler.py | 4 +- src/linkml_transformer/compiler/compiler.py | 15 +++- .../compiler/python_compiler.py | 68 +++++++++++++++++++ .../compiler/r2rml_compiler.py | 4 +- .../compiler/sparql_compiler.py | 4 +- .../compiler/sql_compiler.py | 4 +- .../compiler/sssom_compiler.py | 4 +- .../datamodel/transformer_model.yaml | 2 - .../schema_mapper/schema_mapper.py | 7 +- .../transformer/inference.py | 21 ++++-- .../transformer/object_transformer.py | 6 +- .../transformer/rdflib_transformer.py | 6 -- .../transformer/transformer.py | 4 +- src/linkml_transformer/utils/loaders.py | 22 ++++++ tests/test_cli/test_cli.py | 4 +- tests/test_compiler/__init__.py | 0 tests/test_compiler/test_python_compiler.py | 32 +++++++++ tests/test_datamodel.py | 13 ++-- tests/test_mapper/test_schema_mapper.py | 28 ++++---- .../test_object_transformer.py | 18 ++--- tox.ini | 10 ++- 22 files changed, 216 insertions(+), 75 deletions(-) create mode 100644 src/linkml_transformer/compiler/python_compiler.py create mode 100644 src/linkml_transformer/utils/loaders.py create mode 100644 tests/test_compiler/__init__.py create mode 100644 tests/test_compiler/test_python_compiler.py diff --git a/src/linkml_transformer/cli/cli.py b/src/linkml_transformer/cli/cli.py index b05ac1a..6427d34 100644 --- a/src/linkml_transformer/cli/cli.py +++ b/src/linkml_transformer/cli/cli.py @@ -79,9 +79,7 @@ def map_data( ) tr = ObjectTransformer() tr.source_schemaview = SchemaView(schema) - tr.specification = yaml_loader.load( - transformer_specification, target_class=TransformationSpecification - ) + tr.load_transformer_specification(transformer_specification) with open(input) as file: input_obj = yaml.safe_load(file) tr.index(input_obj, source_type) @@ -114,12 +112,11 @@ def derive_schema(schema, transformer_specification, output, output_format, **kw linkml-tr derive-schema -T transform/personinfo-to-agent.transform.yaml source/personinfo.yaml """ logging.info(f"Transforming {schema} using {transformer_specification}") - tr = SchemaMapper() - tr.source_schemaview = SchemaView(schema) - specification = yaml_loader.load( - transformer_specification, target_class=TransformationSpecification - ) - target_schema = tr.derive_schema(specification) + tr = ObjectTransformer() + tr.load_transformer_specification(transformer_specification) + mapper = SchemaMapper(transformer=tr) + mapper.source_schemaview = SchemaView(schema) + target_schema = mapper.derive_schema() if output: file = open(output, "w", encoding="utf-8") else: diff --git a/src/linkml_transformer/compiler/awk_compiler.py b/src/linkml_transformer/compiler/awk_compiler.py index 4bfdad1..a665889 100644 --- a/src/linkml_transformer/compiler/awk_compiler.py +++ b/src/linkml_transformer/compiler/awk_compiler.py @@ -1,6 +1,6 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler +from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -12,5 +12,5 @@ class AWKCompiler(Compiler): Note: this is only expected to work for flat schemas. """ - def compile(self, specification: TransformationSpecification) -> YAMLRoot: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/compiler.py b/src/linkml_transformer/compiler/compiler.py index 01a97d5..3a24d21 100644 --- a/src/linkml_transformer/compiler/compiler.py +++ b/src/linkml_transformer/compiler/compiler.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from typing import Iterator from linkml_runtime import SchemaView from linkml_runtime.utils.yamlutils import YAMLRoot @@ -8,6 +9,10 @@ TransformationSpecification +@dataclass +class CompiledSpecification: + serialization: str + @dataclass class Compiler(ABC): """ @@ -22,12 +27,18 @@ class Compiler(ABC): source_schemaview: SchemaView = None """A view over the schema describing the source.""" - @abstractmethod - def compile(self, specification: TransformationSpecification) -> YAMLRoot: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: """ Transform source object into an instance of the target class. :param specification: :return: """ + s = "" + for chunk in self._compile_iterator(specification): + s += chunk + return CompiledSpecification(serialization=s) + + def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]: raise NotImplementedError + diff --git a/src/linkml_transformer/compiler/python_compiler.py b/src/linkml_transformer/compiler/python_compiler.py new file mode 100644 index 0000000..c702e52 --- /dev/null +++ b/src/linkml_transformer/compiler/python_compiler.py @@ -0,0 +1,68 @@ +from copy import deepcopy +from typing import Iterator + +from jinja2 import Template +from linkml_runtime.utils.yamlutils import YAMLRoot + +from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification +from linkml_transformer.datamodel.transformer_model import \ + TransformationSpecification, ClassDerivation +from linkml_transformer.transformer.inference import induce_missing_values + +CD_TEMPLATE = """ +{% macro gen_slot_derivation_value(sd, var) -%} +{%- if sd.range -%} +derive_{{ sd.range }}({{ var }}) +{%- else -%} +{{ var }} +{%- endif -%} +{%- endmacro %} +{% macro gen_slot_derivation(sd, force_singlevalued=False) -%} +{%- if not force_singlevalued and sd.populated_from and induced_slots[sd.populated_from].multivalued -%} + [ {{ gen_slot_derivation_value(sd, "x") }} for x in {{ gen_slot_derivation(sd, force_singlevalued=True) }} ] +{%- else -%} + {%- if sd.populated_from -%} + source_object.{{ sd.populated_from }} + {%- elif sd.expr -%} + {{ sd.expr }} + {%- else -%} + None + {%- endif -%} +{%- endif -%} +{%- endmacro %} +def derive_{{ cd.name }}( + source_object: {{ source_module }}.{{ cd.populated_from }} + ) -> {{ target_module }}.{{ cd.name }}: + return {{ cd.populated_from }}( + {%- for sd in cd.slot_derivations.values() %} + {{ sd.name }}={{ gen_slot_derivation(sd) }}, + {%- endfor %} + ) + +""" + + +class PythonCompiler(Compiler): + """ + Compiles a Transformation Specification to Python code. + """ + + def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]: + specification = deepcopy(specification) + induce_missing_values(specification, self.source_schemaview) + for cd in specification.class_derivations.values(): + yield from self._yield_compile_class_derivation(cd) + + def _yield_compile_class_derivation(self, cd: ClassDerivation) -> Iterator[str]: + sv = self.source_schemaview + if cd.populated_from: + populated_from = cd.populated_from + else: + populated_from = cd.name + if populated_from not in sv.all_classes(): + return + induced_slots = {s.name: s for s in sv.class_induced_slots(populated_from)} + t = Template(CD_TEMPLATE) + yield t.render(cd=cd, source_module="src", target_module="tgt", induced_slots=induced_slots, schemaview=sv) + + diff --git a/src/linkml_transformer/compiler/r2rml_compiler.py b/src/linkml_transformer/compiler/r2rml_compiler.py index c477cb4..d16e54e 100644 --- a/src/linkml_transformer/compiler/r2rml_compiler.py +++ b/src/linkml_transformer/compiler/r2rml_compiler.py @@ -1,6 +1,6 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler +from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -11,5 +11,5 @@ class R2RMLCompiler(Compiler): """ - def compile(self, specification: TransformationSpecification) -> YAMLRoot: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sparql_compiler.py b/src/linkml_transformer/compiler/sparql_compiler.py index dcac9b7..654cefe 100644 --- a/src/linkml_transformer/compiler/sparql_compiler.py +++ b/src/linkml_transformer/compiler/sparql_compiler.py @@ -1,6 +1,6 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler +from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -11,5 +11,5 @@ class SPARQLCompiler(Compiler): """ - def compile(self, specification: TransformationSpecification) -> YAMLRoot: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sql_compiler.py b/src/linkml_transformer/compiler/sql_compiler.py index 62cafa9..9f253c4 100644 --- a/src/linkml_transformer/compiler/sql_compiler.py +++ b/src/linkml_transformer/compiler/sql_compiler.py @@ -1,6 +1,6 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler +from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -11,5 +11,5 @@ class SQLCompiler(Compiler): """ - def compile(self, specification: TransformationSpecification) -> YAMLRoot: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sssom_compiler.py b/src/linkml_transformer/compiler/sssom_compiler.py index c9f6d84..21a74c1 100644 --- a/src/linkml_transformer/compiler/sssom_compiler.py +++ b/src/linkml_transformer/compiler/sssom_compiler.py @@ -1,6 +1,6 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler +from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -12,5 +12,5 @@ class SSSOMCompiler(Compiler): Note: SSSOM has less expressivity so this is expected to be highly lossy """ - def compile(self, specification: TransformationSpecification) -> YAMLRoot: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/datamodel/transformer_model.yaml b/src/linkml_transformer/datamodel/transformer_model.yaml index 5f13e3a..a42d745 100644 --- a/src/linkml_transformer/datamodel/transformer_model.yaml +++ b/src/linkml_transformer/datamodel/transformer_model.yaml @@ -175,8 +175,6 @@ classes: range: boolean cast_collection_as: range: CollectionType - #range: - # slot_uri: linkml:range EnumDerivation: is_a: ElementDerivation diff --git a/src/linkml_transformer/schema_mapper/schema_mapper.py b/src/linkml_transformer/schema_mapper/schema_mapper.py index d3e926b..fa1ed07 100644 --- a/src/linkml_transformer/schema_mapper/schema_mapper.py +++ b/src/linkml_transformer/schema_mapper/schema_mapper.py @@ -11,6 +11,7 @@ from linkml_transformer.datamodel.transformer_model import ( ClassDerivation, CopyDirective, TransformationSpecification) +from linkml_transformer.transformer.transformer import Transformer @dataclass @@ -21,12 +22,14 @@ class SchemaMapper: source_schemaview: SchemaView = None + transformer: Transformer = None + source_to_target_class_mappings: Dict[str, List[str]] = field( default_factory=lambda: defaultdict(list) ) def derive_schema( - self, specification: TransformationSpecification + self, specification: Optional[TransformationSpecification] = None ) -> SchemaDefinition: """ Use a transformation specification to generate a target/profile schema from a source schema. @@ -34,6 +37,8 @@ def derive_schema( :param specification: :return: """ + if specification is None: + specification = self.transformer.specification source_schemaview = self.source_schemaview source_schema = source_schemaview.schema target_schema = SchemaDefinition(id=source_schema.id, name=source_schema.name) diff --git a/src/linkml_transformer/transformer/inference.py b/src/linkml_transformer/transformer/inference.py index 0464dec..f72b02f 100644 --- a/src/linkml_transformer/transformer/inference.py +++ b/src/linkml_transformer/transformer/inference.py @@ -1,9 +1,7 @@ -import re - from linkml_runtime import SchemaView -from linkml_transformer.datamodel.transformer_model import ( - SlotDerivation, TransformationSpecification) +from linkml_transformer.datamodel.transformer_model import \ + TransformationSpecification def induce_missing_values( @@ -19,6 +17,8 @@ def induce_missing_values( :return: """ for cd in specification.class_derivations.values(): + if not cd.populated_from: + cd.populated_from = cd.name src_cls_name = cd.populated_from src_cls = source_schemaview.get_class(src_cls_name) for slot_match, directive in cd.copy_directives.items(): @@ -29,3 +29,16 @@ def induce_missing_values( # sd = SlotDerivation(name=sn, populated_from=sn) # print(f"Adding {src_cls_name} . {sd}") # cd.slot_derivations[sd.name] = sd + for cd in specification.class_derivations.values(): + for sd in cd.slot_derivations.values(): + if sd.populated_from is None and sd.expr is None: + sd.populated_from = sd.name + if not sd.range: + if sd.populated_from: + if cd.populated_from not in source_schemaview.all_classes(): + continue + source_induced_slot = source_schemaview.induced_slot(sd.populated_from, cd.populated_from) + source_induced_slot_range = source_induced_slot.range + for range_cd in specification.class_derivations.values(): + if range_cd.populated_from == source_induced_slot_range: + sd.range = range_cd.name diff --git a/src/linkml_transformer/transformer/object_transformer.py b/src/linkml_transformer/transformer/object_transformer.py index 0b0e4d6..0ebdeea 100644 --- a/src/linkml_transformer/transformer/object_transformer.py +++ b/src/linkml_transformer/transformer/object_transformer.py @@ -8,7 +8,7 @@ from pydantic import BaseModel from linkml_transformer.transformer.transformer import OBJECT_TYPE, Transformer -from linkml_transformer.utils.dynamic_object import DynObj, dynamic_object +from linkml_transformer.utils.dynamic_object import dynamic_object DICT_OBJ = Dict[str, Any] @@ -95,13 +95,13 @@ def transform( if not isinstance(source_obj, dict): logger.warning(f"Unexpected: {source_obj} for type {source_type}") return source_obj - source_type_class = sv.get_class(source_type) + # source_type_class = sv.get_class(source_type) class_deriv = self._get_class_derivation(source_type) tgt_attrs = {} for slot_derivation in class_deriv.slot_derivations.values(): v = None source_class_slot = None - target_class_slot = None + # target_class_slot = None if slot_derivation.populated_from: v = source_obj.get(slot_derivation.populated_from, None) source_class_slot = sv.induced_slot( diff --git a/src/linkml_transformer/transformer/rdflib_transformer.py b/src/linkml_transformer/transformer/rdflib_transformer.py index 55025f3..7c6a9f3 100644 --- a/src/linkml_transformer/transformer/rdflib_transformer.py +++ b/src/linkml_transformer/transformer/rdflib_transformer.py @@ -1,10 +1,4 @@ from dataclasses import dataclass -from typing import Type - -import rdflib -from linkml_runtime.dumpers import rdflib_dumper -from linkml_runtime.loaders import rdflib_loader -from linkml_runtime.utils.yamlutils import YAMLRoot from linkml_transformer.transformer.transformer import Transformer diff --git a/src/linkml_transformer/transformer/transformer.py b/src/linkml_transformer/transformer/transformer.py index 4b162b0..2d9e5e7 100644 --- a/src/linkml_transformer/transformer/transformer.py +++ b/src/linkml_transformer/transformer/transformer.py @@ -4,12 +4,11 @@ from dataclasses import dataclass from pathlib import Path from types import ModuleType -from typing import Any, Dict, Optional, Type, Union +from typing import Any, Dict, Optional, Union import yaml from curies import Converter from linkml_runtime import SchemaView -from linkml_runtime.loaders import yaml_loader from linkml_runtime.processing.referencevalidator import ReferenceValidator from linkml_runtime.utils.introspection import package_schemaview from linkml_runtime.utils.yamlutils import YAMLRoot @@ -88,6 +87,7 @@ def load_transformer_specification(self, path: Union[str, Path]): # self.specification = yaml_loader.load(str(path), TransformationSpecification) with open(path) as f: obj = yaml.safe_load(f) + # necessary to expand first normalizer = ReferenceValidator( package_schemaview("linkml_transformer.datamodel.transformer_model") ) diff --git a/src/linkml_transformer/utils/loaders.py b/src/linkml_transformer/utils/loaders.py new file mode 100644 index 0000000..a84e42d --- /dev/null +++ b/src/linkml_transformer/utils/loaders.py @@ -0,0 +1,22 @@ +from pathlib import Path +from typing import Union + +import yaml +from linkml_runtime.processing.referencevalidator import ReferenceValidator +from linkml_runtime.utils.introspection import package_schemaview + +from linkml_transformer.datamodel.transformer_model import TransformationSpecification + + +def load_specification(path: Union[Path, str]) -> TransformationSpecification: + if isinstance(path, Path): + path = str(path) + with open(path) as f: + obj = yaml.safe_load(f) + # necessary to expand first + normalizer = ReferenceValidator( + package_schemaview("linkml_transformer.datamodel.transformer_model") + ) + normalizer.expand_all = True + obj = normalizer.normalize(obj) + return TransformationSpecification(**obj) \ No newline at end of file diff --git a/tests/test_cli/test_cli.py b/tests/test_cli/test_cli.py index 60c611b..d4092e3 100644 --- a/tests/test_cli/test_cli.py +++ b/tests/test_cli/test_cli.py @@ -7,8 +7,8 @@ from linkml_runtime.loaders import yaml_loader from linkml_transformer.cli.cli import main -from tests import (DENORM_SCHEMA, DENORM_SPECIFICATION, FLATTENING_DATA, - NORM_SCHEMA, PERSONINFO_CONTAINER_DATA, PERSONINFO_DATA, +from tests import (DENORM_SPECIFICATION, FLATTENING_DATA, + NORM_SCHEMA, PERSONINFO_CONTAINER_DATA, PERSONINFO_SRC_SCHEMA, PERSONINFO_TGT_SCHEMA, PERSONINFO_TR) diff --git a/tests/test_compiler/__init__.py b/tests/test_compiler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_compiler/test_python_compiler.py b/tests/test_compiler/test_python_compiler.py new file mode 100644 index 0000000..10a0e99 --- /dev/null +++ b/tests/test_compiler/test_python_compiler.py @@ -0,0 +1,32 @@ +import unittest + +from linkml_runtime import SchemaView +from linkml_runtime.dumpers import yaml_dumper +from linkml_runtime.loaders import yaml_loader + +from linkml_transformer.compiler.python_compiler import PythonCompiler +from linkml_transformer.datamodel.transformer_model import * +from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper +from linkml_transformer.transformer.object_transformer import ObjectTransformer +from linkml_transformer.utils.loaders import load_specification +from tests import SCHEMA1, SPECIFICATION + + +class PythonCompilerTestCase(unittest.TestCase): + """ + Tests compilation of a specification to python + """ + + def setUp(self) -> None: + self.compiler = PythonCompiler(source_schemaview=SchemaView(SCHEMA1)) + + def test_compile(self): + spec = load_specification(SPECIFICATION) + pycode = self.compiler.compile(spec) + print(pycode.serialization) + + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_datamodel.py b/tests/test_datamodel.py index b96a071..38e77df 100644 --- a/tests/test_datamodel.py +++ b/tests/test_datamodel.py @@ -1,10 +1,9 @@ -import logging -import os import unittest from linkml_runtime.dumpers import yaml_dumper from linkml_runtime.loaders import yaml_loader +from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.datamodel.transformer_model import * from tests import PERSONINFO_TR @@ -15,15 +14,15 @@ class DatamodelTestCase(unittest.TestCase): """ def setUp(self) -> None: - self.tr_spec = yaml_loader.load( - str(PERSONINFO_TR), target_class=TransformationSpecification - ) + tr = ObjectTransformer() + tr.load_transformer_specification(PERSONINFO_TR) + self.tr_spec = tr.specification def test_datamodel(self): """checks loading/retrieval""" tr_spec = self.tr_spec - print(yaml_dumper.dumps(tr_spec)) - self.assertEqual(1, 1) + # print(tr_spec.json()) + self.assertNotEqual("", tr_spec.json()) if __name__ == "__main__": diff --git a/tests/test_mapper/test_schema_mapper.py b/tests/test_mapper/test_schema_mapper.py index 2badda5..d413b3d 100644 --- a/tests/test_mapper/test_schema_mapper.py +++ b/tests/test_mapper/test_schema_mapper.py @@ -6,6 +6,7 @@ from linkml_transformer.datamodel.transformer_model import * from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper +from linkml_transformer.transformer.object_transformer import ObjectTransformer from tests import SCHEMA1, SPECIFICATION @@ -15,17 +16,20 @@ class SchemaMapperTestCase(unittest.TestCase): """ def setUp(self) -> None: - tr = SchemaMapper() - tr.source_schemaview = SchemaView(SCHEMA1) - self.specification = yaml_loader.load( - SPECIFICATION, target_class=TransformationSpecification - ) - self.tr = tr + mapper = SchemaMapper() + tr = ObjectTransformer() + self.source_schemaview = SchemaView(SCHEMA1) + mapper.source_schemaview = self.source_schemaview + tr.source_schemaview = self.source_schemaview + tr.load_transformer_specification(SPECIFICATION) + self.transformer = tr + self.specification = tr.specification + self.mapper = mapper def test_derive_schema(self): """tests deriving a schema from a specification and a source""" - tr = self.tr - target_schema = tr.derive_schema(self.specification) + mapper = self.mapper + target_schema = mapper.derive_schema(self.specification) cases = [ ( "Agent", @@ -51,15 +55,15 @@ def test_derive_schema(self): def test_derive_null(self): """tests empty spec limit case""" - tr = self.tr - specification = TransformationSpecification("test") + tr = self.mapper + specification = TransformationSpecification(id="test") target_schema = tr.derive_schema(specification) self.assertEqual([], list(target_schema.classes.values())) def test_derive_partial(self): """tests empty spec limit case""" - tr = self.tr - specification = TransformationSpecification("test") + tr = self.mapper + specification = TransformationSpecification(id="test") derivations = [ ClassDerivation(name="Agent", populated_from="Person"), ] diff --git a/tests/test_transformer/test_object_transformer.py b/tests/test_transformer/test_object_transformer.py index 49d10dc..8cd2f75 100644 --- a/tests/test_transformer/test_object_transformer.py +++ b/tests/test_transformer/test_object_transformer.py @@ -30,9 +30,7 @@ def setUp(self) -> None: tr = ObjectTransformer() tr.source_schemaview = SchemaView(str(PERSONINFO_SRC_SCHEMA)) tr.target_schemaview = SchemaView(str(PERSONINFO_TGT_SCHEMA)) - tr.specification = yaml_loader.load( - str(PERSONINFO_TR), target_class=TransformationSpecification - ) + tr.load_transformer_specification(PERSONINFO_TR) self.tr = tr def test_transform_simple_dict(self): @@ -206,9 +204,7 @@ def test_denormalized_transform_dict(self): tr = ObjectTransformer() tr.source_schemaview = SchemaView(NORM_SCHEMA) tr.target_schemaview = SchemaView(DENORM_SCHEMA) - tr.specification = yaml_loader.load( - DENORM_SPECIFICATION, target_class=TransformationSpecification - ) + tr.load_transformer_specification(DENORM_SPECIFICATION) mset = yaml.safe_load(open(str(FLATTENING_DATA))) self.assertEqual( mset["mappings"], [{"subject": "X:1", "object": "Y:1", "predicate": "P:1"}] @@ -243,9 +239,7 @@ def test_denormalized_object_transform(self): tr.source_schemaview = SchemaView(NORM_SCHEMA) tr.target_schemaview = SchemaView(DENORM_SCHEMA) tr.target_module = sssom_tgt_dm - tr.specification = yaml_loader.load( - DENORM_SPECIFICATION, target_class=TransformationSpecification - ) + tr.load_transformer_specification(DENORM_SPECIFICATION) mset: sssom_src_dm.MappingSet = yaml_loader.load( str(FLATTENING_DATA), target_class=sssom_src_dm.MappingSet ) @@ -304,10 +298,10 @@ def mk(mv: bool, explicit: bool = False): source_schema = mk(source_multivalued) target_schema = mk(target_multivalued, explicit) - specification = TransformationSpecification("test") - cd = ClassDerivation(class_name, populated_from=class_name) + specification = TransformationSpecification(id="test") + cd = ClassDerivation(name=class_name, populated_from=class_name) specification.class_derivations[class_name] = cd - sd = SlotDerivation(att_name, populated_from=att_name) + sd = SlotDerivation(name=att_name, populated_from=att_name) if explicit: sd.cast_collection_as = ( CollectionType.MultiValued diff --git a/tox.ini b/tox.ini index 0a666cf..4122c13 100644 --- a/tox.ini +++ b/tox.ini @@ -25,8 +25,12 @@ deps = flake8 flake8-black flake8-isort + flake8-bandit + flake8-bugbear + flake8-colors + pep8-naming commands = - flake8 src/linkml_transformer/transformer tests/*py + flake8 src/linkml_transformer/transformer description = Run the flake8 tool with several plugins (bandit, docstrings, import order, pep8 naming). ######################### @@ -40,8 +44,8 @@ exclude = ignore = E203 W503 - C901 # needs code change so ignoring for now. - E731 # needs code change so ignoring for now. + C901 + E731 max-line-length = 120 max-complexity = 13 import-order-style = pycharm From 1324004cc2483aef6623c17bbb77ff0bfca26add Mon Sep 17 00:00:00 2001 From: cmungall Date: Mon, 1 May 2023 17:37:45 -0700 Subject: [PATCH 05/15] resolved commented code block --- src/linkml_transformer/transformer/inference.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/linkml_transformer/transformer/inference.py b/src/linkml_transformer/transformer/inference.py index f72b02f..1034cb1 100644 --- a/src/linkml_transformer/transformer/inference.py +++ b/src/linkml_transformer/transformer/inference.py @@ -25,10 +25,6 @@ def induce_missing_values( for sn in source_schemaview.class_slots(src_cls.name): if sn in cd.slot_derivations: continue - # if slot_match == "*" or re.match(slot_match, sn): - # sd = SlotDerivation(name=sn, populated_from=sn) - # print(f"Adding {src_cls_name} . {sd}") - # cd.slot_derivations[sd.name] = sd for cd in specification.class_derivations.values(): for sd in cd.slot_derivations.values(): if sd.populated_from is None and sd.expr is None: From ab0f3aa29280fde7ef64bf22a41add5774351195 Mon Sep 17 00:00:00 2001 From: cmungall Date: Mon, 1 May 2023 18:02:03 -0700 Subject: [PATCH 06/15] tidying test --- .../compiler/awk_compiler.py | 7 ++- src/linkml_transformer/compiler/compiler.py | 10 +++- .../compiler/python_compiler.py | 21 ++++--- .../compiler/r2rml_compiler.py | 7 ++- .../compiler/sparql_compiler.py | 7 ++- .../compiler/sql_compiler.py | 7 ++- .../compiler/sssom_compiler.py | 7 ++- .../transformer/inference.py | 4 +- src/linkml_transformer/utils/loaders.py | 5 +- tests/test_cli/test_cli.py | 6 +- tests/test_compiler/test_python_compiler.py | 2 - tests/test_datamodel.py | 2 +- .../test_transformer_examples.py | 58 +++++-------------- 13 files changed, 72 insertions(+), 71 deletions(-) diff --git a/src/linkml_transformer/compiler/awk_compiler.py b/src/linkml_transformer/compiler/awk_compiler.py index a665889..36e3a22 100644 --- a/src/linkml_transformer/compiler/awk_compiler.py +++ b/src/linkml_transformer/compiler/awk_compiler.py @@ -1,6 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification +from linkml_transformer.compiler.compiler import (CompiledSpecification, + Compiler) from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -12,5 +13,7 @@ class AWKCompiler(Compiler): Note: this is only expected to work for flat schemas. """ - def compile(self, specification: TransformationSpecification) -> CompiledSpecification: + def compile( + self, specification: TransformationSpecification + ) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/compiler.py b/src/linkml_transformer/compiler/compiler.py index 3a24d21..ed89780 100644 --- a/src/linkml_transformer/compiler/compiler.py +++ b/src/linkml_transformer/compiler/compiler.py @@ -13,6 +13,7 @@ class CompiledSpecification: serialization: str + @dataclass class Compiler(ABC): """ @@ -27,7 +28,9 @@ class Compiler(ABC): source_schemaview: SchemaView = None """A view over the schema describing the source.""" - def compile(self, specification: TransformationSpecification) -> CompiledSpecification: + def compile( + self, specification: TransformationSpecification + ) -> CompiledSpecification: """ Transform source object into an instance of the target class. @@ -39,6 +42,7 @@ def compile(self, specification: TransformationSpecification) -> CompiledSpecifi s += chunk return CompiledSpecification(serialization=s) - def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]: + def _compile_iterator( + self, specification: TransformationSpecification + ) -> Iterator[str]: raise NotImplementedError - diff --git a/src/linkml_transformer/compiler/python_compiler.py b/src/linkml_transformer/compiler/python_compiler.py index c702e52..74a7e8a 100644 --- a/src/linkml_transformer/compiler/python_compiler.py +++ b/src/linkml_transformer/compiler/python_compiler.py @@ -4,9 +4,10 @@ from jinja2 import Template from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification, ClassDerivation +from linkml_transformer.compiler.compiler import (CompiledSpecification, + Compiler) +from linkml_transformer.datamodel.transformer_model import ( + ClassDerivation, TransformationSpecification) from linkml_transformer.transformer.inference import induce_missing_values CD_TEMPLATE = """ @@ -47,7 +48,9 @@ class PythonCompiler(Compiler): Compiles a Transformation Specification to Python code. """ - def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]: + def _compile_iterator( + self, specification: TransformationSpecification + ) -> Iterator[str]: specification = deepcopy(specification) induce_missing_values(specification, self.source_schemaview) for cd in specification.class_derivations.values(): @@ -63,6 +66,10 @@ def _yield_compile_class_derivation(self, cd: ClassDerivation) -> Iterator[str]: return induced_slots = {s.name: s for s in sv.class_induced_slots(populated_from)} t = Template(CD_TEMPLATE) - yield t.render(cd=cd, source_module="src", target_module="tgt", induced_slots=induced_slots, schemaview=sv) - - + yield t.render( + cd=cd, + source_module="src", + target_module="tgt", + induced_slots=induced_slots, + schemaview=sv, + ) diff --git a/src/linkml_transformer/compiler/r2rml_compiler.py b/src/linkml_transformer/compiler/r2rml_compiler.py index d16e54e..f2935bb 100644 --- a/src/linkml_transformer/compiler/r2rml_compiler.py +++ b/src/linkml_transformer/compiler/r2rml_compiler.py @@ -1,6 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification +from linkml_transformer.compiler.compiler import (CompiledSpecification, + Compiler) from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -11,5 +12,7 @@ class R2RMLCompiler(Compiler): """ - def compile(self, specification: TransformationSpecification) -> CompiledSpecification: + def compile( + self, specification: TransformationSpecification + ) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sparql_compiler.py b/src/linkml_transformer/compiler/sparql_compiler.py index 654cefe..e76204b 100644 --- a/src/linkml_transformer/compiler/sparql_compiler.py +++ b/src/linkml_transformer/compiler/sparql_compiler.py @@ -1,6 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification +from linkml_transformer.compiler.compiler import (CompiledSpecification, + Compiler) from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -11,5 +12,7 @@ class SPARQLCompiler(Compiler): """ - def compile(self, specification: TransformationSpecification) -> CompiledSpecification: + def compile( + self, specification: TransformationSpecification + ) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sql_compiler.py b/src/linkml_transformer/compiler/sql_compiler.py index 9f253c4..a9e2461 100644 --- a/src/linkml_transformer/compiler/sql_compiler.py +++ b/src/linkml_transformer/compiler/sql_compiler.py @@ -1,6 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification +from linkml_transformer.compiler.compiler import (CompiledSpecification, + Compiler) from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -11,5 +12,7 @@ class SQLCompiler(Compiler): """ - def compile(self, specification: TransformationSpecification) -> CompiledSpecification: + def compile( + self, specification: TransformationSpecification + ) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sssom_compiler.py b/src/linkml_transformer/compiler/sssom_compiler.py index 21a74c1..17ef53f 100644 --- a/src/linkml_transformer/compiler/sssom_compiler.py +++ b/src/linkml_transformer/compiler/sssom_compiler.py @@ -1,6 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import Compiler, CompiledSpecification +from linkml_transformer.compiler.compiler import (CompiledSpecification, + Compiler) from linkml_transformer.datamodel.transformer_model import \ TransformationSpecification @@ -12,5 +13,7 @@ class SSSOMCompiler(Compiler): Note: SSSOM has less expressivity so this is expected to be highly lossy """ - def compile(self, specification: TransformationSpecification) -> CompiledSpecification: + def compile( + self, specification: TransformationSpecification + ) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/transformer/inference.py b/src/linkml_transformer/transformer/inference.py index 1034cb1..dcb7f70 100644 --- a/src/linkml_transformer/transformer/inference.py +++ b/src/linkml_transformer/transformer/inference.py @@ -33,7 +33,9 @@ def induce_missing_values( if sd.populated_from: if cd.populated_from not in source_schemaview.all_classes(): continue - source_induced_slot = source_schemaview.induced_slot(sd.populated_from, cd.populated_from) + source_induced_slot = source_schemaview.induced_slot( + sd.populated_from, cd.populated_from + ) source_induced_slot_range = source_induced_slot.range for range_cd in specification.class_derivations.values(): if range_cd.populated_from == source_induced_slot_range: diff --git a/src/linkml_transformer/utils/loaders.py b/src/linkml_transformer/utils/loaders.py index a84e42d..57522bd 100644 --- a/src/linkml_transformer/utils/loaders.py +++ b/src/linkml_transformer/utils/loaders.py @@ -5,7 +5,8 @@ from linkml_runtime.processing.referencevalidator import ReferenceValidator from linkml_runtime.utils.introspection import package_schemaview -from linkml_transformer.datamodel.transformer_model import TransformationSpecification +from linkml_transformer.datamodel.transformer_model import \ + TransformationSpecification def load_specification(path: Union[Path, str]) -> TransformationSpecification: @@ -19,4 +20,4 @@ def load_specification(path: Union[Path, str]) -> TransformationSpecification: ) normalizer.expand_all = True obj = normalizer.normalize(obj) - return TransformationSpecification(**obj) \ No newline at end of file + return TransformationSpecification(**obj) diff --git a/tests/test_cli/test_cli.py b/tests/test_cli/test_cli.py index d4092e3..81d72dc 100644 --- a/tests/test_cli/test_cli.py +++ b/tests/test_cli/test_cli.py @@ -7,9 +7,9 @@ from linkml_runtime.loaders import yaml_loader from linkml_transformer.cli.cli import main -from tests import (DENORM_SPECIFICATION, FLATTENING_DATA, - NORM_SCHEMA, PERSONINFO_CONTAINER_DATA, - PERSONINFO_SRC_SCHEMA, PERSONINFO_TGT_SCHEMA, PERSONINFO_TR) +from tests import (DENORM_SPECIFICATION, FLATTENING_DATA, NORM_SCHEMA, + PERSONINFO_CONTAINER_DATA, PERSONINFO_SRC_SCHEMA, + PERSONINFO_TGT_SCHEMA, PERSONINFO_TR) class TestCommandLineInterface(unittest.TestCase): diff --git a/tests/test_compiler/test_python_compiler.py b/tests/test_compiler/test_python_compiler.py index 10a0e99..b973fb5 100644 --- a/tests/test_compiler/test_python_compiler.py +++ b/tests/test_compiler/test_python_compiler.py @@ -26,7 +26,5 @@ def test_compile(self): print(pycode.serialization) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_datamodel.py b/tests/test_datamodel.py index 38e77df..8dbb4b4 100644 --- a/tests/test_datamodel.py +++ b/tests/test_datamodel.py @@ -3,8 +3,8 @@ from linkml_runtime.dumpers import yaml_dumper from linkml_runtime.loaders import yaml_loader -from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.datamodel.transformer_model import * +from linkml_transformer.transformer.object_transformer import ObjectTransformer from tests import PERSONINFO_TR diff --git a/tests/test_transformer/test_transformer_examples.py b/tests/test_transformer/test_transformer_examples.py index 6028acc..8719ad3 100644 --- a/tests/test_transformer/test_transformer_examples.py +++ b/tests/test_transformer/test_transformer_examples.py @@ -3,20 +3,16 @@ import yaml from linkml_transformer.transformer.object_transformer import ObjectTransformer -from linkml_transformer.utils.dynamic_object import dynamic_object from linkml_transformer.utils.multi_file_transformer import \ MultiFileTransformer from tests import EXAMPLE_DIR -EXAMPLES = [ - ( - "measurements", - "quantity_value", - "qv-to-scalar", - "PersonQuantityValue-001", - 172.0, - ), - ("measurements", "quantity_value", "qv-to-scalar", "PersonQuantityValue-002", None), +EXAMPLE_PROJECTS = [ + "measurements", + "flattening", + "personinfo_basic", + "type_coercion", + "biolink", ] @@ -34,50 +30,28 @@ class TransformerExamplesTestCase(unittest.TestCase): - target/{SourceClassName}-{LocalId}.yaml :: expected output data """ - def test_examples(self): - """ - Tests transforming a Person object from s1 to an Agent object in s2 - """ - for example in EXAMPLES: - folder, src, spec, data, expected = example - path = EXAMPLE_DIR / folder - tr = ObjectTransformer() - tr.load_source_schema(path / "source" / f"{src}.yaml") - tr.load_transformer_specification( - path / "transform" / f"{spec}.transform.yaml" - ) - input_obj = yaml.safe_load(open(str(path / "data" / f"{data}.yaml"))) - target_obj = tr.transform(input_obj) - # target_obj = dynamic_object(target_obj, "Person") - self.assertEqual(expected, target_obj["height_in_cm"]) - def test_all(self): """ - Iterates through all examples - :return: + Iterates through all examples. + + This uses the MultiFileProcessor in test_mode - if the outputs differ + then the test will fail """ mft = MultiFileTransformer() - dirs = [ - "measurements", - "flattening", - "personinfo_basic", - "type_coercion", - "biolink", - ] - for dir in dirs: + for dir in EXAMPLE_PROJECTS: full_dir = EXAMPLE_DIR / dir instructions = mft.infer_instructions(full_dir) - # print(yaml.dump(instructions.dict())) - obj = mft.process_instructions(instructions, full_dir, test_mode=True) + mft.process_instructions(instructions, full_dir, test_mode=True) + @unittest.skip("Uncomment this to regenerate examples") def test_regenerate(self): """ - Use this to regenerate test examples + Use this to regenerate test examples. """ mft = MultiFileTransformer() - dirs = ["biolink"] + dirs = EXAMPLE_PROJECTS for dir in dirs: full_dir = EXAMPLE_DIR / dir instructions = mft.infer_instructions(full_dir) # print(yaml.dump(instructions.dict())) - obj = mft.process_instructions(instructions, full_dir, test_mode=False) + mft.process_instructions(instructions, full_dir, test_mode=False) From 86064507fc821cec1afbe2aface06c378f83d95f Mon Sep 17 00:00:00 2001 From: cmungall Date: Mon, 1 May 2023 18:02:43 -0700 Subject: [PATCH 07/15] removing remaining prints --- src/linkml_transformer/utils/multi_file_transformer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/linkml_transformer/utils/multi_file_transformer.py b/src/linkml_transformer/utils/multi_file_transformer.py index f53dcc1..714852c 100644 --- a/src/linkml_transformer/utils/multi_file_transformer.py +++ b/src/linkml_transformer/utils/multi_file_transformer.py @@ -128,7 +128,6 @@ def infer_instructions(self, root_directory: Union[str, Path]) -> Instructions: raise ValueError(f"Ambiguous: {transform_file}") src, target_schema_base = matches.group(1, 2) if src not in input_schema: - print(f"SKIP {input_schema}") continue instructions.transformations.append(tr) data_files = glob.glob( From 7c5806cb421912aa77b7afca838e27c42fbd591b Mon Sep 17 00:00:00 2001 From: cmungall Date: Mon, 1 May 2023 18:07:41 -0700 Subject: [PATCH 08/15] add missing --- .../biolink/output/gene-001.transformed.yaml | 1 + .../input/examples/biolink/target/target.yaml | 309 ++++++++++++++++++ .../output/MappingSet-001.transformed.yaml | 6 + .../PersonQuantityValue-001.transformed.yaml | 2 + .../PersonQuantityValue-002.transformed.yaml | 2 + .../target/quantity_value_flat.yaml | 16 + .../output/Container-001.transformed.yaml | 37 +++ .../output/Person-001.transformed.yaml | 14 + .../type_coercion/data/MyRecord-001.yaml | 4 + .../output/MyRecord-001.transformed.yaml | 10 + .../examples/type_coercion/target/target.yaml | 51 +++ 11 files changed, 452 insertions(+) create mode 100644 tests/input/examples/biolink/output/gene-001.transformed.yaml create mode 100644 tests/input/examples/biolink/target/target.yaml create mode 100644 tests/input/examples/flattening/output/MappingSet-001.transformed.yaml create mode 100644 tests/input/examples/measurements/output/PersonQuantityValue-001.transformed.yaml create mode 100644 tests/input/examples/measurements/output/PersonQuantityValue-002.transformed.yaml create mode 100644 tests/input/examples/measurements/target/quantity_value_flat.yaml create mode 100644 tests/input/examples/personinfo_basic/output/Container-001.transformed.yaml create mode 100644 tests/input/examples/personinfo_basic/output/Person-001.transformed.yaml create mode 100644 tests/input/examples/type_coercion/data/MyRecord-001.yaml create mode 100644 tests/input/examples/type_coercion/output/MyRecord-001.transformed.yaml create mode 100644 tests/input/examples/type_coercion/target/target.yaml diff --git a/tests/input/examples/biolink/output/gene-001.transformed.yaml b/tests/input/examples/biolink/output/gene-001.transformed.yaml new file mode 100644 index 0000000..2e24149 --- /dev/null +++ b/tests/input/examples/biolink/output/gene-001.transformed.yaml @@ -0,0 +1 @@ +symbol: foo diff --git a/tests/input/examples/biolink/target/target.yaml b/tests/input/examples/biolink/target/target.yaml new file mode 100644 index 0000000..d50e68c --- /dev/null +++ b/tests/input/examples/biolink/target/target.yaml @@ -0,0 +1,309 @@ +name: Biolink-Model +id: https://w3id.org/biolink/biolink-model +default_prefix: https://w3id.org/biolink/biolink-model/ +classes: + NamedThing: + name: NamedThing + description: a databased entity or concept/class + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - BFO:0000001 + - WIKIDATA:Q35120 + - UMLSSG:OBJC + - STY:T071 + - dcid:Thing + attributes: + id: + name: id + description: A unique identifier for an entity. Must be either a CURIE shorthand + for a URI or a complete URI + in_subset: + - translator_minimal + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - AGRKB:primaryId + - gff3:ID + - gpi:DB_Object_ID + domain: entity + identifier: true + required: true + name: + name: name + description: A human-readable name for an attribute or entity. + in_subset: + - translator_minimal + - samples + from_schema: https://w3id.org/biolink/biolink-model + aliases: + - label + - display name + - title + exact_mappings: + - gff3:Name + - gpi:DB_Object_Name + narrow_mappings: + - dct:title + - WIKIDATA_PROPERTY:P1476 + domain: entity + slot_uri: rdfs:label + range: label type + category: + name: category + description: "Name of the high level ontology class in which this entity is\ + \ categorized. Corresponds to the label for the biolink entity type class.\n\ + \ * In a neo4j database this MAY correspond to the neo4j label tag.\n *\ + \ In an RDF database it should be a biolink model class URI.\nThis field\ + \ is multi-valued. It should include values for ancestors of the biolink\ + \ class; for example, a protein such as Shh would have category values `biolink:Protein`,\ + \ `biolink:GeneProduct`, `biolink:MolecularEntity`, ...\nIn an RDF database,\ + \ nodes will typically have an rdf:type triples. This can be to the most\ + \ specific biolink class, or potentially to a class more specific than something\ + \ in biolink. For example, a sequence feature `f` may have a rdf:type assertion\ + \ to a SO class such as TF_binding_site, which is more specific than anything\ + \ in biolink. Here we would have categories {biolink:GenomicEntity, biolink:MolecularEntity,\ + \ biolink:NamedThing}" + in_subset: + - translator_minimal + from_schema: https://w3id.org/biolink/biolink-model + is_a: type + domain: entity + multivalued: true + designates_type: true + is_class_field: true + range: category type + Gene: + name: Gene + id_prefixes: + - NCBIGene + - ENSEMBL + - HGNC + - MGI + - ZFIN + - dictyBase + - WB + - WormBase + - FB + - RGD + - SGD + - PomBase + - OMIM + - KEGG.GENE + - UMLS + - Xenbase + - AspGD + description: A region (or regions) that includes all of the sequence elements + necessary to encode a functional transcript. A gene locus may include regulatory + regions, transcribed regions and/or other functional sequence regions. + in_subset: + - translator_minimal + - model_organism_database + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - SO:0000704 + - SIO:010035 + - WIKIDATA:Q7187 + - dcid:Gene + narrow_mappings: + - bioschemas:Gene + broad_mappings: + - NCIT:C45822 + is_a: NamedThing + attributes: + symbol: + name: symbol + description: Symbol for a particular thing + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - AGRKB:symbol + - gpi:DB_Object_Symbol + is_a: node property + domain: named thing + Disease: + name: Disease + id_prefixes: + - MONDO + - DOID + - OMIM + - OMIM.PS + - orphanet + - EFO + - UMLS + - MESH + - MEDDRA + - NCIT + - SNOMEDCT + - medgen + - ICD10 + - ICD9 + - KEGG.DISEASE + - HP + - MP + description: A disorder of structure or function, especially one that produces + specific signs, phenotypes or symptoms or that affects a specific location + and is not simply a direct result of physical injury. A disposition to undergo + pathological processes that exists in an organism because of one or more disorders + in that organism. + in_subset: + - model_organism_database + from_schema: https://w3id.org/biolink/biolink-model + aliases: + - condition + - disorder + - medical condition + exact_mappings: + - MONDO:0000001 + - DOID:4 + - NCIT:C2991 + - WIKIDATA:Q12136 + - SIO:010299 + - UMLSSG:DISO + - STY:T047 + - dcid:Disease + narrow_mappings: + - STY:T019 + - STY:T020 + - STY:T048 + - STY:T049 + - STY:T190 + - STY:T191 + - MONDO:0042489 + is_a: NamedThing + PhenotypicFeature: + name: PhenotypicFeature + id_prefixes: + - HP + - EFO + - NCIT + - UMLS + - MEDDRA + - MP + - ZP + - UPHENO + - APO + - FBcv + - WBPhenotype + - SNOMEDCT + - MESH + - XPO + - FYPO + - TO + description: A combination of entity and quality that makes up a phenotyping statement. + An observable characteristic of an individual resulting from the interaction + of its genotype with its molecular and physical environment. + examples: + - value: MP:0001262 + description: decreased body weight + in_subset: + - model_organism_database + from_schema: https://w3id.org/biolink/biolink-model + aliases: + - sign + - symptom + - phenotype + - trait + - endophenotype + exact_mappings: + - UPHENO:0001001 + - SIO:010056 + - WIKIDATA:Q104053 + - UMLS:C4021819 + - NCIT:C16977 + - SNOMEDCT:8116006 + - MESH:D010641 + narrow_mappings: + - STY:T184 + - WIKIDATA:Q169872 + - WIKIDATA:Q25203551 + - ZP:00000000 + - FBcv:0001347 + - HP:0000118 + - MP:0000001 + - WBPhenotype:0000886 + - XPO:00000000 + - FYPO:0000001 + - APO:0000017 + - TO:0000387 + broad_mappings: + - BFO:0000019 + - PATO:0000001 + is_a: NamedThing + Association: + name: Association + description: A typed association between two entities, supported by evidence + comments: + - This is roughly the model used by biolink and ontobio at the moment + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - OBAN:association + - rdf:Statement + - owl:Axiom + attributes: + subject: + name: subject + local_names: + ga4gh: + local_name_source: ga4gh + local_name_value: annotation subject + neo4j: + local_name_source: neo4j + local_name_value: node with outgoing relationship + description: connects an association to the subject of the association. For + example, in a gene-to-phenotype association, the gene is subject and phenotype + is object. + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - owl:annotatedSource + - OBAN:association_has_subject + is_a: association slot + domain: association + slot_uri: rdf:subject + range: named thing + required: true + predicate: + name: predicate + local_names: + ga4gh: + local_name_source: ga4gh + local_name_value: annotation predicate + translator: + local_name_source: translator + local_name_value: predicate + description: A high-level grouping for the relationship type. AKA minimal + predicate. This is analogous to category for nodes. + notes: + - Has a value from the Biolink related_to hierarchy. In RDF, this corresponds + to rdf:predicate and in Neo4j this corresponds to the relationship type. + The convention is for an edge label in snake_case form. For example, biolink:related_to, + biolink:causes, biolink:treats + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - owl:annotatedProperty + - OBAN:association_has_predicate + is_a: association slot + domain: association + slot_uri: rdf:predicate + range: predicate type + required: true + object: + name: object + local_names: + ga4gh: + local_name_source: ga4gh + local_name_value: descriptor + neo4j: + local_name_source: neo4j + local_name_value: node with incoming relationship + description: connects an association to the object of the association. For + example, in a gene-to-phenotype association, the gene is subject and phenotype + is object. + from_schema: https://w3id.org/biolink/biolink-model + exact_mappings: + - owl:annotatedTarget + - OBAN:association_has_object + is_a: association slot + domain: association + slot_uri: rdf:object + range: named thing + required: true + GeneToPhenotypicFeatureAssociation: + name: GeneToPhenotypicFeatureAssociation diff --git a/tests/input/examples/flattening/output/MappingSet-001.transformed.yaml b/tests/input/examples/flattening/output/MappingSet-001.transformed.yaml new file mode 100644 index 0000000..5c165a1 --- /dev/null +++ b/tests/input/examples/flattening/output/MappingSet-001.transformed.yaml @@ -0,0 +1,6 @@ +mappings: +- subject_id: X:1 + subject_name: x1 + object_id: Y:1 + object_name: y1 + predicate_id: P:1 diff --git a/tests/input/examples/measurements/output/PersonQuantityValue-001.transformed.yaml b/tests/input/examples/measurements/output/PersonQuantityValue-001.transformed.yaml new file mode 100644 index 0000000..e0f5c81 --- /dev/null +++ b/tests/input/examples/measurements/output/PersonQuantityValue-001.transformed.yaml @@ -0,0 +1,2 @@ +id: P:1 +height_in_cm: 172.0 diff --git a/tests/input/examples/measurements/output/PersonQuantityValue-002.transformed.yaml b/tests/input/examples/measurements/output/PersonQuantityValue-002.transformed.yaml new file mode 100644 index 0000000..6026b67 --- /dev/null +++ b/tests/input/examples/measurements/output/PersonQuantityValue-002.transformed.yaml @@ -0,0 +1,2 @@ +id: P:2 +height_in_cm: null diff --git a/tests/input/examples/measurements/target/quantity_value_flat.yaml b/tests/input/examples/measurements/target/quantity_value_flat.yaml new file mode 100644 index 0000000..865241a --- /dev/null +++ b/tests/input/examples/measurements/target/quantity_value_flat.yaml @@ -0,0 +1,16 @@ +name: quantity_value +id: https://w3id.org/linkml/examples/quantity_value +default_prefix: https://w3id.org/linkml/examples/quantity_value/ +classes: + Person: + name: Person + from_schema: https://w3id.org/linkml/examples/quantity_value + attributes: + id: + name: id + from_schema: https://w3id.org/linkml/examples/quantity_value + identifier: true + owner: Person + height_in_cm: + name: height_in_cm + tree_root: true diff --git a/tests/input/examples/personinfo_basic/output/Container-001.transformed.yaml b/tests/input/examples/personinfo_basic/output/Container-001.transformed.yaml new file mode 100644 index 0000000..5f7627b --- /dev/null +++ b/tests/input/examples/personinfo_basic/output/Container-001.transformed.yaml @@ -0,0 +1,37 @@ +agents: +- id: P:001 + label: fred bloggs + age: 33 years + primary_email: fred.bloggs@example.com + gender: null + current_address: + address_of: foo + street: 1 oak street + city: foo + has_familial_relationships: + - type: SIBLING_OF + related_to: P:002 + - type: CHILD_OF + related_to: P:003 +- id: P:002 + label: Alison Wu + age: null + primary_email: null + gender: null + current_address: null + has_familial_relationships: + - type: SIBLING_OF + related_to: P:001 + - type: CHILD_OF + related_to: P:003 +- id: P:003 + label: Mr. Blobby + age: null + primary_email: null + gender: null + current_address: null + has_familial_relationships: + - type: PARENT_OF + related_to: P:001 + - type: PARENT_OF + related_to: P:002 diff --git a/tests/input/examples/personinfo_basic/output/Person-001.transformed.yaml b/tests/input/examples/personinfo_basic/output/Person-001.transformed.yaml new file mode 100644 index 0000000..035ce86 --- /dev/null +++ b/tests/input/examples/personinfo_basic/output/Person-001.transformed.yaml @@ -0,0 +1,14 @@ +id: P:001 +label: fred bloggs +age: 33 years +primary_email: fred.bloggs@example.com +gender: null +current_address: + address_of: foo + street: 1 oak street + city: foo +has_familial_relationships: +- type: SIBLING_OF + related_to: P:002 +- type: CHILD_OF + related_to: P:003 diff --git a/tests/input/examples/type_coercion/data/MyRecord-001.yaml b/tests/input/examples/type_coercion/data/MyRecord-001.yaml new file mode 100644 index 0000000..6341fc7 --- /dev/null +++ b/tests/input/examples/type_coercion/data/MyRecord-001.yaml @@ -0,0 +1,4 @@ +id: P:1 +name: my name +age_as_integer: 33 +url: https://example.org/foo diff --git a/tests/input/examples/type_coercion/output/MyRecord-001.transformed.yaml b/tests/input/examples/type_coercion/output/MyRecord-001.transformed.yaml new file mode 100644 index 0000000..27771bb --- /dev/null +++ b/tests/input/examples/type_coercion/output/MyRecord-001.transformed.yaml @@ -0,0 +1,10 @@ +id: P:1 +id_as_string: P:1 +id_as_url: https://example.org/person/1 +url: https://example.org/foo +url_as_string: https://example.org/foo +url_as_curie: example:foo +name: my name +age_as_integer: 33 +age_as_float: 33.0 +age_as_string: '33' diff --git a/tests/input/examples/type_coercion/target/target.yaml b/tests/input/examples/type_coercion/target/target.yaml new file mode 100644 index 0000000..7ede85f --- /dev/null +++ b/tests/input/examples/type_coercion/target/target.yaml @@ -0,0 +1,51 @@ +name: quantity_value +id: https://w3id.org/linkml/examples/type_coercion +default_prefix: https://w3id.org/linkml/examples/type_coercion/ +classes: + MyRecord: + name: MyRecord + from_schema: https://w3id.org/linkml/examples/type_coercion + attributes: + id: + name: id + from_schema: https://w3id.org/linkml/examples/type_coercion + identifier: true + owner: MyRecord + range: uriorcurie + id_as_string: + name: id_as_string + from_schema: https://w3id.org/linkml/examples/type_coercion + identifier: true + owner: MyRecord + range: uriorcurie + url: + name: url + from_schema: https://w3id.org/linkml/examples/type_coercion + owner: MyRecord + range: uri + url_as_string: + name: url_as_string + from_schema: https://w3id.org/linkml/examples/type_coercion + owner: MyRecord + range: uri + name: + name: name + from_schema: https://w3id.org/linkml/examples/type_coercion + owner: MyRecord + range: string + age_as_integer: + name: age_as_integer + from_schema: https://w3id.org/linkml/examples/type_coercion + owner: MyRecord + range: integer + age_as_float: + name: age_as_float + from_schema: https://w3id.org/linkml/examples/type_coercion + owner: MyRecord + range: integer + age_as_string: + name: age_as_string + from_schema: https://w3id.org/linkml/examples/type_coercion + owner: MyRecord + range: integer + tree_root: true From 2dcbbd417741f06908551ea1eb448241e9b144b9 Mon Sep 17 00:00:00 2001 From: cmungall Date: Mon, 1 May 2023 18:12:46 -0700 Subject: [PATCH 09/15] lint --- src/linkml_transformer/transformer/inference.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/linkml_transformer/transformer/inference.py b/src/linkml_transformer/transformer/inference.py index dcb7f70..bebfa11 100644 --- a/src/linkml_transformer/transformer/inference.py +++ b/src/linkml_transformer/transformer/inference.py @@ -19,12 +19,6 @@ def induce_missing_values( for cd in specification.class_derivations.values(): if not cd.populated_from: cd.populated_from = cd.name - src_cls_name = cd.populated_from - src_cls = source_schemaview.get_class(src_cls_name) - for slot_match, directive in cd.copy_directives.items(): - for sn in source_schemaview.class_slots(src_cls.name): - if sn in cd.slot_derivations: - continue for cd in specification.class_derivations.values(): for sd in cd.slot_derivations.values(): if sd.populated_from is None and sd.expr is None: From 624329dcc735ac9575fe4f72feba934d23920f5a Mon Sep 17 00:00:00 2001 From: cmungall Date: Mon, 1 May 2023 18:16:29 -0700 Subject: [PATCH 10/15] lint --- pyproject.toml | 10 ++ src/linkml_transformer/cli/cli.py | 7 +- .../compiler/awk_compiler.py | 10 +- src/linkml_transformer/compiler/compiler.py | 11 +- .../compiler/python_compiler.py | 11 +- .../compiler/r2rml_compiler.py | 10 +- .../compiler/sparql_compiler.py | 10 +- .../compiler/sql_compiler.py | 10 +- .../compiler/sssom_compiler.py | 10 +- .../datamodel/specification_view.py | 4 +- .../datamodel/transformer_model.py | 36 ++---- src/linkml_transformer/importer/importer.py | 3 +- .../schema_mapper/schema_mapper.py | 23 ++-- .../transformer/inference.py | 3 +- .../transformer/object_transformer.py | 40 ++----- .../transformer/transformer.py | 15 ++- src/linkml_transformer/utils/inverter.py | 3 +- src/linkml_transformer/utils/loaders.py | 3 +- .../utils/multi_file_transformer.py | 28 ++--- .../flattening/model/denormalized_model.py | 27 +++-- .../flattening/model/normalized_model.py | 31 +++--- .../personinfo_basic/model/agent_model.py | 101 +++++++---------- .../model/personinfo_model.py | 104 +++++++----------- tests/test_cli/test_cli.py | 12 +- .../test_object_transformer.py | 40 +++---- .../test_transformer_examples.py | 3 +- tests/test_utils/test_dynamic_object.py | 13 ++- 27 files changed, 241 insertions(+), 337 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 88fe68f..cb8f3dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,13 @@ jupyter = "^1.0.0" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.black] +line-length = 100 +target-version = ["py39", "py310"] + +[tool.isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = true +reverse_relative = true diff --git a/src/linkml_transformer/cli/cli.py b/src/linkml_transformer/cli/cli.py index 6427d34..d2a3a1b 100644 --- a/src/linkml_transformer/cli/cli.py +++ b/src/linkml_transformer/cli/cli.py @@ -13,8 +13,7 @@ from linkml_runtime.dumpers import yaml_dumper from linkml_runtime.loaders import yaml_loader -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.datamodel.transformer_model import TransformationSpecification from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper from linkml_transformer.transformer.object_transformer import ObjectTransformer @@ -74,9 +73,7 @@ def map_data( **kwargs, ): """Map data in a source schema using a transformation.""" - logging.info( - f"Transforming {input} conforming to {schema} using {transformer_specification}" - ) + logging.info(f"Transforming {input} conforming to {schema} using {transformer_specification}") tr = ObjectTransformer() tr.source_schemaview = SchemaView(schema) tr.load_transformer_specification(transformer_specification) diff --git a/src/linkml_transformer/compiler/awk_compiler.py b/src/linkml_transformer/compiler/awk_compiler.py index 36e3a22..9bb11be 100644 --- a/src/linkml_transformer/compiler/awk_compiler.py +++ b/src/linkml_transformer/compiler/awk_compiler.py @@ -1,9 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import (CompiledSpecification, - Compiler) -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler +from linkml_transformer.datamodel.transformer_model import TransformationSpecification class AWKCompiler(Compiler): @@ -13,7 +11,5 @@ class AWKCompiler(Compiler): Note: this is only expected to work for flat schemas. """ - def compile( - self, specification: TransformationSpecification - ) -> CompiledSpecification: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/compiler.py b/src/linkml_transformer/compiler/compiler.py index ed89780..2f46079 100644 --- a/src/linkml_transformer/compiler/compiler.py +++ b/src/linkml_transformer/compiler/compiler.py @@ -5,8 +5,7 @@ from linkml_runtime import SchemaView from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.datamodel.transformer_model import TransformationSpecification @dataclass @@ -28,9 +27,7 @@ class Compiler(ABC): source_schemaview: SchemaView = None """A view over the schema describing the source.""" - def compile( - self, specification: TransformationSpecification - ) -> CompiledSpecification: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: """ Transform source object into an instance of the target class. @@ -42,7 +39,5 @@ def compile( s += chunk return CompiledSpecification(serialization=s) - def _compile_iterator( - self, specification: TransformationSpecification - ) -> Iterator[str]: + def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/python_compiler.py b/src/linkml_transformer/compiler/python_compiler.py index 74a7e8a..528fec8 100644 --- a/src/linkml_transformer/compiler/python_compiler.py +++ b/src/linkml_transformer/compiler/python_compiler.py @@ -4,10 +4,11 @@ from jinja2 import Template from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import (CompiledSpecification, - Compiler) +from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler from linkml_transformer.datamodel.transformer_model import ( - ClassDerivation, TransformationSpecification) + ClassDerivation, + TransformationSpecification, +) from linkml_transformer.transformer.inference import induce_missing_values CD_TEMPLATE = """ @@ -48,9 +49,7 @@ class PythonCompiler(Compiler): Compiles a Transformation Specification to Python code. """ - def _compile_iterator( - self, specification: TransformationSpecification - ) -> Iterator[str]: + def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]: specification = deepcopy(specification) induce_missing_values(specification, self.source_schemaview) for cd in specification.class_derivations.values(): diff --git a/src/linkml_transformer/compiler/r2rml_compiler.py b/src/linkml_transformer/compiler/r2rml_compiler.py index f2935bb..6f5a889 100644 --- a/src/linkml_transformer/compiler/r2rml_compiler.py +++ b/src/linkml_transformer/compiler/r2rml_compiler.py @@ -1,9 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import (CompiledSpecification, - Compiler) -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler +from linkml_transformer.datamodel.transformer_model import TransformationSpecification class R2RMLCompiler(Compiler): @@ -12,7 +10,5 @@ class R2RMLCompiler(Compiler): """ - def compile( - self, specification: TransformationSpecification - ) -> CompiledSpecification: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sparql_compiler.py b/src/linkml_transformer/compiler/sparql_compiler.py index e76204b..4799754 100644 --- a/src/linkml_transformer/compiler/sparql_compiler.py +++ b/src/linkml_transformer/compiler/sparql_compiler.py @@ -1,9 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import (CompiledSpecification, - Compiler) -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler +from linkml_transformer.datamodel.transformer_model import TransformationSpecification class SPARQLCompiler(Compiler): @@ -12,7 +10,5 @@ class SPARQLCompiler(Compiler): """ - def compile( - self, specification: TransformationSpecification - ) -> CompiledSpecification: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sql_compiler.py b/src/linkml_transformer/compiler/sql_compiler.py index a9e2461..5d8f0e7 100644 --- a/src/linkml_transformer/compiler/sql_compiler.py +++ b/src/linkml_transformer/compiler/sql_compiler.py @@ -1,9 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import (CompiledSpecification, - Compiler) -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler +from linkml_transformer.datamodel.transformer_model import TransformationSpecification class SQLCompiler(Compiler): @@ -12,7 +10,5 @@ class SQLCompiler(Compiler): """ - def compile( - self, specification: TransformationSpecification - ) -> CompiledSpecification: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/compiler/sssom_compiler.py b/src/linkml_transformer/compiler/sssom_compiler.py index 17ef53f..fd58531 100644 --- a/src/linkml_transformer/compiler/sssom_compiler.py +++ b/src/linkml_transformer/compiler/sssom_compiler.py @@ -1,9 +1,7 @@ from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import (CompiledSpecification, - Compiler) -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler +from linkml_transformer.datamodel.transformer_model import TransformationSpecification class SSSOMCompiler(Compiler): @@ -13,7 +11,5 @@ class SSSOMCompiler(Compiler): Note: SSSOM has less expressivity so this is expected to be highly lossy """ - def compile( - self, specification: TransformationSpecification - ) -> CompiledSpecification: + def compile(self, specification: TransformationSpecification) -> CompiledSpecification: raise NotImplementedError diff --git a/src/linkml_transformer/datamodel/specification_view.py b/src/linkml_transformer/datamodel/specification_view.py index f06cfc6..b09cf3c 100644 --- a/src/linkml_transformer/datamodel/specification_view.py +++ b/src/linkml_transformer/datamodel/specification_view.py @@ -1,7 +1,9 @@ from typing import List from linkml_transformer.datamodel.transformer_model import ( - ElementDerivation, TransformationSpecification) + ElementDerivation, + TransformationSpecification, +) @dataclass diff --git a/src/linkml_transformer/datamodel/transformer_model.py b/src/linkml_transformer/datamodel/transformer_model.py index 7013aa5..ce3b39a 100644 --- a/src/linkml_transformer/datamodel/transformer_model.py +++ b/src/linkml_transformer/datamodel/transformer_model.py @@ -78,9 +78,7 @@ class ElementDerivation(ConfiguredBaseModel): name: str = Field(None, description="""Name of the element in the target schema""") copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) - overrides: Optional[Any] = Field( - None, description="""overrides source schema slots""" - ) + overrides: Optional[Any] = Field(None, description="""overrides source schema slots""") is_a: Optional[str] = Field(None) mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) value_mappings: Optional[Dict[str, KeyVal]] = Field( @@ -112,9 +110,7 @@ class ClassDerivation(ElementDerivation): slot_derivations: Optional[Dict[str, SlotDerivation]] = Field(default_factory=dict) name: str = Field(None, description="""Name of the element in the target schema""") copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) - overrides: Optional[Any] = Field( - None, description="""overrides source schema slots""" - ) + overrides: Optional[Any] = Field(None, description="""overrides source schema slots""") is_a: Optional[str] = Field(None) mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) value_mappings: Optional[Dict[str, KeyVal]] = Field( @@ -137,9 +133,7 @@ class AliasedClass(ConfiguredBaseModel): """ alias: str = Field(None, description="""name of the class to be aliased""") - class_named: Optional[str] = Field( - None, description="""local alias for the class""" - ) + class_named: Optional[str] = Field(None, description="""local alias for the class""") class SlotDerivation(ElementDerivation): @@ -162,9 +156,7 @@ class SlotDerivation(ElementDerivation): type_designator: Optional[bool] = Field(None) cast_collection_as: Optional[CollectionType] = Field(None) copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) - overrides: Optional[Any] = Field( - None, description="""overrides source schema slots""" - ) + overrides: Optional[Any] = Field(None, description="""overrides source schema slots""") is_a: Optional[str] = Field(None) mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) value_mappings: Optional[Dict[str, KeyVal]] = Field( @@ -193,16 +185,12 @@ class EnumDerivation(ElementDerivation): description="""An expression to be evaluated on the source object to derive the target slot. Should be specified using the LinkML expression language.""", ) hide: Optional[bool] = Field(None, description="""True if this is suppressed""") - permissible_value_derivations: Optional[ - Dict[str, PermissibleValueDerivation] - ] = Field( + permissible_value_derivations: Optional[Dict[str, PermissibleValueDerivation]] = Field( default_factory=dict, description="""Instructions on how to derive a set of PVs in the target schema""", ) copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) - overrides: Optional[Any] = Field( - None, description="""overrides source schema slots""" - ) + overrides: Optional[Any] = Field(None, description="""overrides source schema slots""") is_a: Optional[str] = Field(None) mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) value_mappings: Optional[Dict[str, KeyVal]] = Field( @@ -226,14 +214,10 @@ class PermissibleValueDerivation(ElementDerivation): name: str = Field(None, description="""Target permissible value text""") expr: Optional[str] = Field(None) - populated_from: Optional[str] = Field( - None, description="""Source permissible value""" - ) + populated_from: Optional[str] = Field(None, description="""Source permissible value""") hide: Optional[bool] = Field(None) copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) - overrides: Optional[Any] = Field( - None, description="""overrides source schema slots""" - ) + overrides: Optional[Any] = Field(None, description="""overrides source schema slots""") is_a: Optional[str] = Field(None) mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) value_mappings: Optional[Dict[str, KeyVal]] = Field( @@ -253,9 +237,7 @@ class PermissibleValueDerivation(ElementDerivation): class PrefixDerivation(ElementDerivation): name: str = Field(None, description="""Name of the element in the target schema""") copy_directives: Optional[Dict[str, CopyDirective]] = Field(default_factory=dict) - overrides: Optional[Any] = Field( - None, description="""overrides source schema slots""" - ) + overrides: Optional[Any] = Field(None, description="""overrides source schema slots""") is_a: Optional[str] = Field(None) mixins: Optional[Dict[str, ElementDerivation]] = Field(default_factory=dict) value_mappings: Optional[Dict[str, KeyVal]] = Field( diff --git a/src/linkml_transformer/importer/importer.py b/src/linkml_transformer/importer/importer.py index 77cb20b..e02912f 100644 --- a/src/linkml_transformer/importer/importer.py +++ b/src/linkml_transformer/importer/importer.py @@ -5,8 +5,7 @@ from linkml_runtime import SchemaView from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.datamodel.transformer_model import TransformationSpecification @dataclass diff --git a/src/linkml_transformer/schema_mapper/schema_mapper.py b/src/linkml_transformer/schema_mapper/schema_mapper.py index fa1ed07..aa3df18 100644 --- a/src/linkml_transformer/schema_mapper/schema_mapper.py +++ b/src/linkml_transformer/schema_mapper/schema_mapper.py @@ -5,12 +5,19 @@ from typing import Dict, List, Optional from linkml_runtime import SchemaView -from linkml_runtime.linkml_model import (ClassDefinition, ClassDefinitionName, - Element, SchemaDefinition, - SlotDefinition) +from linkml_runtime.linkml_model import ( + ClassDefinition, + ClassDefinitionName, + Element, + SchemaDefinition, + SlotDefinition, +) from linkml_transformer.datamodel.transformer_model import ( - ClassDerivation, CopyDirective, TransformationSpecification) + ClassDerivation, + CopyDirective, + TransformationSpecification, +) from linkml_transformer.transformer.transformer import Transformer @@ -89,12 +96,8 @@ def _derive_slot(self, slot_derivation) -> SlotDefinition: def _rewire_class(self, class_definition: ClassDefinition): if class_definition.is_a: - class_definition.is_a = self._rewire_parent( - class_definition, class_definition.is_a - ) - mixins = [ - self._rewire_parent(class_definition, m) for m in class_definition.mixins - ] + class_definition.is_a = self._rewire_parent(class_definition, class_definition.is_a) + mixins = [self._rewire_parent(class_definition, m) for m in class_definition.mixins] class_definition.mixins = [m for m in mixins if m is not None] def _rewire_parent( diff --git a/src/linkml_transformer/transformer/inference.py b/src/linkml_transformer/transformer/inference.py index bebfa11..57f489a 100644 --- a/src/linkml_transformer/transformer/inference.py +++ b/src/linkml_transformer/transformer/inference.py @@ -1,7 +1,6 @@ from linkml_runtime import SchemaView -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.datamodel.transformer_model import TransformationSpecification def induce_missing_values( diff --git a/src/linkml_transformer/transformer/object_transformer.py b/src/linkml_transformer/transformer/object_transformer.py index 0ebdeea..bc9a923 100644 --- a/src/linkml_transformer/transformer/object_transformer.py +++ b/src/linkml_transformer/transformer/object_transformer.py @@ -36,24 +36,14 @@ def index(self, source_obj: Any, target: str = None): if isinstance(source_obj, dict): if target is None: [target] = [ - c.name - for c in self.source_schemaview.all_classes().values() - if c.tree_root + c.name for c in self.source_schemaview.all_classes().values() if c.tree_root ] if target is None: - raise ValueError( - f"target must be passed if source_obj is dict: {source_obj}" - ) - source_obj_typed = dynamic_object( - source_obj, self.source_schemaview, target - ) - self.object_index = ObjectIndex( - source_obj_typed, schemaview=self.source_schemaview - ) + raise ValueError(f"target must be passed if source_obj is dict: {source_obj}") + source_obj_typed = dynamic_object(source_obj, self.source_schemaview, target) + self.object_index = ObjectIndex(source_obj_typed, schemaview=self.source_schemaview) else: - self.object_index = ObjectIndex( - source_obj, schemaview=self.source_schemaview - ) + self.object_index = ObjectIndex(source_obj, schemaview=self.source_schemaview) def transform( self, @@ -104,9 +94,7 @@ def transform( # target_class_slot = None if slot_derivation.populated_from: v = source_obj.get(slot_derivation.populated_from, None) - source_class_slot = sv.induced_slot( - slot_derivation.populated_from, source_type - ) + source_class_slot = sv.induced_slot(slot_derivation.populated_from, source_type) logger.debug( f"Pop slot {slot_derivation.name} => {v} using {slot_derivation.populated_from} // {source_obj}" ) @@ -135,15 +123,9 @@ def transform( source_class_slot_range = source_class_slot.range if source_class_slot.multivalued: if isinstance(v, list): - v = [ - self.transform(v1, source_class_slot_range, target_range) - for v1 in v - ] + v = [self.transform(v1, source_class_slot_range, target_range) for v1 in v] elif isinstance(v, dict): - v = [ - self.transform(v1, source_class_slot_range, target_range) - for v1 in v - ] + v = [self.transform(v1, source_class_slot_range, target_range) for v1 in v] else: v = [v] else: @@ -154,9 +136,9 @@ def transform( and not isinstance(v, list) ): v = [v] - if self._coerce_to_singlevalued( - slot_derivation, class_deriv - ) and isinstance(v, list): + if self._coerce_to_singlevalued(slot_derivation, class_deriv) and isinstance( + v, list + ): if len(v) > 1: raise ValueError(f"Cannot coerce multiple values {v}") if len(v) == 0: diff --git a/src/linkml_transformer/transformer/transformer.py b/src/linkml_transformer/transformer/transformer.py index 2d9e5e7..eff6e71 100644 --- a/src/linkml_transformer/transformer/transformer.py +++ b/src/linkml_transformer/transformer/transformer.py @@ -15,8 +15,11 @@ from pydantic import BaseModel from linkml_transformer.datamodel.transformer_model import ( - ClassDerivation, CollectionType, SlotDerivation, - TransformationSpecification) + ClassDerivation, + CollectionType, + SlotDerivation, + TransformationSpecification, +) from linkml_transformer.transformer.inference import induce_missing_values logger = logging.getLogger(__name__) @@ -113,9 +116,7 @@ def _get_class_derivation(self, target_class_name) -> ClassDerivation: ] logger.debug(f"Target class derivs={matching_tgt_class_derivs}") if len(matching_tgt_class_derivs) != 1: - raise ValueError( - f"Could not find what to derive from a source {target_class_name}" - ) + raise ValueError(f"Could not find what to derive from a source {target_class_name}") return matching_tgt_class_derivs[0] def _coerce_to_multivalued( @@ -149,9 +150,7 @@ def curie_converter(self) -> Converter: if not self._curie_converter: self._curie_converter = Converter([]) for prefix in self.source_schemaview.schema.prefixes.values(): - self._curie_converter.add_prefix( - prefix.prefix_prefix, prefix.prefix_reference - ) + self._curie_converter.add_prefix(prefix.prefix_prefix, prefix.prefix_reference) for prefix in self.specification.prefixes.values(): self._curie_converter.add_prefix(prefix.key, prefix.value) return self._curie_converter diff --git a/src/linkml_transformer/utils/inverter.py b/src/linkml_transformer/utils/inverter.py index 060775e..03917c5 100644 --- a/src/linkml_transformer/utils/inverter.py +++ b/src/linkml_transformer/utils/inverter.py @@ -1,5 +1,4 @@ -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.datamodel.transformer_model import TransformationSpecification def invert_transformation_specification( diff --git a/src/linkml_transformer/utils/loaders.py b/src/linkml_transformer/utils/loaders.py index 57522bd..d126189 100644 --- a/src/linkml_transformer/utils/loaders.py +++ b/src/linkml_transformer/utils/loaders.py @@ -5,8 +5,7 @@ from linkml_runtime.processing.referencevalidator import ReferenceValidator from linkml_runtime.utils.introspection import package_schemaview -from linkml_transformer.datamodel.transformer_model import \ - TransformationSpecification +from linkml_transformer.datamodel.transformer_model import TransformationSpecification def load_specification(path: Union[Path, str]) -> TransformationSpecification: diff --git a/src/linkml_transformer/utils/multi_file_transformer.py b/src/linkml_transformer/utils/multi_file_transformer.py index 714852c..89ce93c 100644 --- a/src/linkml_transformer/utils/multi_file_transformer.py +++ b/src/linkml_transformer/utils/multi_file_transformer.py @@ -60,9 +60,7 @@ class MultiFileTransformer: """ source_schema_directory_base: str = field(default_factory=lambda: "source") - transform_specification_directory_base: str = field( - default_factory=lambda: "transform" - ) + transform_specification_directory_base: str = field(default_factory=lambda: "transform") source_data_directory_base: str = field(default_factory=lambda: "data") target_schema_directory_base: str = field(default_factory=lambda: "target") target_data_directory_base: str = field(default_factory=lambda: "output") @@ -131,27 +129,19 @@ def infer_instructions(self, root_directory: Union[str, Path]) -> Instructions: continue instructions.transformations.append(tr) data_files = glob.glob( - os.path.join( - str(root_directory / self.source_data_directory_base), "*.yaml" - ) + os.path.join(str(root_directory / self.source_data_directory_base), "*.yaml") ) for data_file in data_files: if target_schema_base and target_schema_base not in data_file: continue - target_data = str( - Path(self.target_data_directory_base) / Path(data_file).name - ) + target_data = str(Path(self.target_data_directory_base) / Path(data_file).name) target_data = target_data.replace(".yaml", ".transformed.yaml") step = Step(source_data=data_file, target_data=target_data) step.source_class = Path(data_file).stem.split("-")[-2] tr.steps.append(step) - target_schemas = glob.glob( - os.path.join(str(target_schema_directory), "*.yaml") - ) + target_schemas = glob.glob(os.path.join(str(target_schema_directory), "*.yaml")) if len(target_schemas) > 1: - target_schemas = [ - s for s in target_schemas if target_schema_base in s - ] + target_schemas = [s for s in target_schemas if target_schema_base in s] if len(target_schemas) != 1: raise ValueError( f"Could not determine target schema from: {target_schemas}" @@ -159,9 +149,7 @@ def infer_instructions(self, root_directory: Union[str, Path]) -> Instructions: if target_schemas: tr.target_schema = target_schemas[0] else: - tr.target_schema = str( - Path(target_schema_directory) / "target.yaml" - ) + tr.target_schema = str(Path(target_schema_directory) / "target.yaml") if not tr.steps: raise ValueError(f"Could not infer steps from {data_files}") if not tr.transformation_specification: @@ -209,9 +197,7 @@ def process_instructions( for step in tr.steps: input_obj = yaml.safe_load(open(str(root_directory / step.source_data))) transformer.index(input_obj, step.source_class) - target_obj = transformer.transform( - input_obj, source_type=step.source_class - ) + target_obj = transformer.transform(input_obj, source_type=step.source_class) if step.target_data: out_path = output_directory / step.target_data out_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/input/examples/flattening/model/denormalized_model.py b/tests/input/examples/flattening/model/denormalized_model.py index 4b2a02c..67bf314 100644 --- a/tests/input/examples/flattening/model/denormalized_model.py +++ b/tests/input/examples/flattening/model/denormalized_model.py @@ -13,18 +13,26 @@ from typing import Any, ClassVar, Dict, List, Optional, Union from jsonasobj2 import JsonObj, as_dict -from linkml_runtime.linkml_model.meta import (EnumDefinition, PermissibleValue, - PvFormulaOptions) +from linkml_runtime.linkml_model.meta import ( + EnumDefinition, + PermissibleValue, + PvFormulaOptions, +) from linkml_runtime.linkml_model.types import String from linkml_runtime.utils.curienamespace import CurieNamespace -from linkml_runtime.utils.dataclass_extensions_376 import \ - dataclasses_init_fn_with_kwargs +from linkml_runtime.utils.dataclass_extensions_376 import ( + dataclasses_init_fn_with_kwargs, +) from linkml_runtime.utils.enumerations import EnumDefinitionImpl from linkml_runtime.utils.formatutils import camelcase, sfx, underscore from linkml_runtime.utils.metamodelcore import bnode, empty_dict, empty_list from linkml_runtime.utils.slot import Slot -from linkml_runtime.utils.yamlutils import (YAMLRoot, extended_float, - extended_int, extended_str) +from linkml_runtime.utils.yamlutils import ( + YAMLRoot, + extended_float, + extended_int, + extended_str, +) from rdflib import Namespace, URIRef metamodel_version = "1.7.0" @@ -58,16 +66,13 @@ class MappingSet(YAMLRoot): class_name: ClassVar[str] = "MappingSet" class_model_uri: ClassVar[URIRef] = MAPPINGS.MappingSet - mappings: Optional[ - Union[Union[dict, "Mapping"], List[Union[dict, "Mapping"]]] - ] = empty_list() + mappings: Optional[Union[Union[dict, "Mapping"], List[Union[dict, "Mapping"]]]] = empty_list() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.mappings, list): self.mappings = [self.mappings] if self.mappings is not None else [] self.mappings = [ - v if isinstance(v, Mapping) else Mapping(**as_dict(v)) - for v in self.mappings + v if isinstance(v, Mapping) else Mapping(**as_dict(v)) for v in self.mappings ] super().__post_init__(**kwargs) diff --git a/tests/input/examples/flattening/model/normalized_model.py b/tests/input/examples/flattening/model/normalized_model.py index 6193acf..eea8b37 100644 --- a/tests/input/examples/flattening/model/normalized_model.py +++ b/tests/input/examples/flattening/model/normalized_model.py @@ -13,18 +13,26 @@ from typing import Any, ClassVar, Dict, List, Optional, Union from jsonasobj2 import JsonObj, as_dict -from linkml_runtime.linkml_model.meta import (EnumDefinition, PermissibleValue, - PvFormulaOptions) +from linkml_runtime.linkml_model.meta import ( + EnumDefinition, + PermissibleValue, + PvFormulaOptions, +) from linkml_runtime.linkml_model.types import String from linkml_runtime.utils.curienamespace import CurieNamespace -from linkml_runtime.utils.dataclass_extensions_376 import \ - dataclasses_init_fn_with_kwargs +from linkml_runtime.utils.dataclass_extensions_376 import ( + dataclasses_init_fn_with_kwargs, +) from linkml_runtime.utils.enumerations import EnumDefinitionImpl from linkml_runtime.utils.formatutils import camelcase, sfx, underscore from linkml_runtime.utils.metamodelcore import bnode, empty_dict, empty_list from linkml_runtime.utils.slot import Slot -from linkml_runtime.utils.yamlutils import (YAMLRoot, extended_float, - extended_int, extended_str) +from linkml_runtime.utils.yamlutils import ( + YAMLRoot, + extended_float, + extended_int, + extended_str, +) from rdflib import Namespace, URIRef metamodel_version = "1.7.0" @@ -61,9 +69,7 @@ class MappingSet(YAMLRoot): class_name: ClassVar[str] = "MappingSet" class_model_uri: ClassVar[URIRef] = MAPPINGS.MappingSet - mappings: Optional[ - Union[Union[dict, "Mapping"], List[Union[dict, "Mapping"]]] - ] = empty_list() + mappings: Optional[Union[Union[dict, "Mapping"], List[Union[dict, "Mapping"]]]] = empty_list() entities: Optional[ Union[ Dict[Union[str, EntityId], Union[dict, "Entity"]], @@ -75,8 +81,7 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.mappings, list): self.mappings = [self.mappings] if self.mappings is not None else [] self.mappings = [ - v if isinstance(v, Mapping) else Mapping(**as_dict(v)) - for v in self.mappings + v if isinstance(v, Mapping) else Mapping(**as_dict(v)) for v in self.mappings ] self._normalize_inlined_as_dict( @@ -205,8 +210,6 @@ class slots: model_uri=MAPPINGS.mappingSet__entities, domain=None, range=Optional[ - Union[ - Dict[Union[str, EntityId], Union[dict, Entity]], List[Union[dict, Entity]] - ] + Union[Dict[Union[str, EntityId], Union[dict, Entity]], List[Union[dict, Entity]]] ], ) diff --git a/tests/input/examples/personinfo_basic/model/agent_model.py b/tests/input/examples/personinfo_basic/model/agent_model.py index 7282e05..88576c7 100644 --- a/tests/input/examples/personinfo_basic/model/agent_model.py +++ b/tests/input/examples/personinfo_basic/model/agent_model.py @@ -13,19 +13,32 @@ from typing import Any, ClassVar, Dict, List, Optional, Union from jsonasobj2 import JsonObj, as_dict -from linkml_runtime.linkml_model.meta import (EnumDefinition, PermissibleValue, - PvFormulaOptions) +from linkml_runtime.linkml_model.meta import ( + EnumDefinition, + PermissibleValue, + PvFormulaOptions, +) from linkml_runtime.linkml_model.types import Boolean, Date, Float, String from linkml_runtime.utils.curienamespace import CurieNamespace -from linkml_runtime.utils.dataclass_extensions_376 import \ - dataclasses_init_fn_with_kwargs +from linkml_runtime.utils.dataclass_extensions_376 import ( + dataclasses_init_fn_with_kwargs, +) from linkml_runtime.utils.enumerations import EnumDefinitionImpl from linkml_runtime.utils.formatutils import camelcase, sfx, underscore -from linkml_runtime.utils.metamodelcore import (Bool, XSDDate, bnode, - empty_dict, empty_list) +from linkml_runtime.utils.metamodelcore import ( + Bool, + XSDDate, + bnode, + empty_dict, + empty_list, +) from linkml_runtime.utils.slot import Slot -from linkml_runtime.utils.yamlutils import (YAMLRoot, extended_float, - extended_int, extended_str) +from linkml_runtime.utils.yamlutils import ( + YAMLRoot, + extended_float, + extended_int, + extended_str, +) from rdflib import Namespace, URIRef metamodel_version = "1.7.0" @@ -42,9 +55,7 @@ ROR = CurieNamespace("ROR", "http://example.org/ror/") FAMREL = CurieNamespace("famrel", "https://example.org/FamilialRelations#") LINKML = CurieNamespace("linkml", "https://w3id.org/linkml/") -PERSONINFO = CurieNamespace( - "personinfo", "https://w3id.org/linkml/examples/personinfo/" -) +PERSONINFO = CurieNamespace("personinfo", "https://w3id.org/linkml/examples/personinfo/") PROV = CurieNamespace("prov", "http://www.w3.org/ns/prov#") RDF = CurieNamespace("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#") RDFS = CurieNamespace("rdfs", "http://www.w3.org/2000/01/rdf-schema#") @@ -174,16 +185,12 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self.gender is not None and not isinstance(self.gender, GenderType): self.gender = GenderType(self.gender) - if self.current_address is not None and not isinstance( - self.current_address, Address - ): + if self.current_address is not None and not isinstance(self.current_address, Address): self.current_address = Address(**as_dict(self.current_address)) if not isinstance(self.has_employment_history, list): self.has_employment_history = ( - [self.has_employment_history] - if self.has_employment_history is not None - else [] + [self.has_employment_history] if self.has_employment_history is not None else [] ) self.has_employment_history = [ v if isinstance(v, EmploymentEvent) else EmploymentEvent(**as_dict(v)) @@ -197,17 +204,13 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): else [] ) self.has_familial_relationships = [ - v - if isinstance(v, FamilialRelationship) - else FamilialRelationship(**as_dict(v)) + v if isinstance(v, FamilialRelationship) else FamilialRelationship(**as_dict(v)) for v in self.has_familial_relationships ] if not isinstance(self.has_medical_history, list): self.has_medical_history = ( - [self.has_medical_history] - if self.has_medical_history is not None - else [] + [self.has_medical_history] if self.has_medical_history is not None else [] ) self.has_medical_history = [ v if isinstance(v, MedicalEvent) else MedicalEvent(**as_dict(v)) @@ -273,17 +276,13 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.id, OrganizationId): self.id = OrganizationId(self.id) - if self.mission_statement is not None and not isinstance( - self.mission_statement, str - ): + if self.mission_statement is not None and not isinstance(self.mission_statement, str): self.mission_statement = str(self.mission_statement) if self.founding_date is not None and not isinstance(self.founding_date, str): self.founding_date = str(self.founding_date) - if self.founding_location is not None and not isinstance( - self.founding_location, PlaceId - ): + if self.founding_location is not None and not isinstance(self.founding_location, PlaceId): self.founding_location = PlaceId(self.founding_location) if not isinstance(self.aliases, list): @@ -367,14 +366,10 @@ class Event(YAMLRoot): is_current: Optional[Union[bool, Bool]] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self.started_at_time is not None and not isinstance( - self.started_at_time, XSDDate - ): + if self.started_at_time is not None and not isinstance(self.started_at_time, XSDDate): self.started_at_time = XSDDate(self.started_at_time) - if self.ended_at_time is not None and not isinstance( - self.ended_at_time, XSDDate - ): + if self.ended_at_time is not None and not isinstance(self.ended_at_time, XSDDate): self.ended_at_time = XSDDate(self.ended_at_time) if self.duration is not None and not isinstance(self.duration, float): @@ -461,14 +456,10 @@ class Relationship(YAMLRoot): type: Optional[str] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self.started_at_time is not None and not isinstance( - self.started_at_time, XSDDate - ): + if self.started_at_time is not None and not isinstance(self.started_at_time, XSDDate): self.started_at_time = XSDDate(self.started_at_time) - if self.ended_at_time is not None and not isinstance( - self.ended_at_time, XSDDate - ): + if self.ended_at_time is not None and not isinstance(self.ended_at_time, XSDDate): self.ended_at_time = XSDDate(self.ended_at_time) if self.related_to is not None and not isinstance(self.related_to, str): @@ -518,9 +509,7 @@ class EmploymentEvent(Event): employed_at: Optional[Union[str, OrganizationId]] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self.employed_at is not None and not isinstance( - self.employed_at, OrganizationId - ): + if self.employed_at is not None and not isinstance(self.employed_at, OrganizationId): self.employed_at = OrganizationId(self.employed_at) super().__post_init__(**kwargs) @@ -543,14 +532,10 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self.in_location is not None and not isinstance(self.in_location, PlaceId): self.in_location = PlaceId(self.in_location) - if self.diagnosis is not None and not isinstance( - self.diagnosis, DiagnosisConcept - ): + if self.diagnosis is not None and not isinstance(self.diagnosis, DiagnosisConcept): self.diagnosis = DiagnosisConcept(**as_dict(self.diagnosis)) - if self.procedure is not None and not isinstance( - self.procedure, ProcedureConcept - ): + if self.procedure is not None and not isinstance(self.procedure, ProcedureConcept): self.procedure = ProcedureConcept(**as_dict(self.procedure)) super().__post_init__(**kwargs) @@ -616,9 +601,7 @@ class FamilialRelationshipType(EnumDefinitionImpl): @classmethod def _addvals(cls): - setattr( - cls, "parent of", PermissibleValue(text="parent of", meaning=FAMREL["02"]) - ) + setattr(cls, "parent of", PermissibleValue(text="parent of", meaning=FAMREL["02"])) class GenderType(EnumDefinitionImpl): @@ -758,9 +741,7 @@ class slots: curie=PERSONINFO.curie("has_employment_history"), model_uri=PERSONINFO.has_employment_history, domain=None, - range=Optional[ - Union[Union[dict, EmploymentEvent], List[Union[dict, EmploymentEvent]]] - ], + range=Optional[Union[Union[dict, EmploymentEvent], List[Union[dict, EmploymentEvent]]]], ) slots.has_medical_history = Slot( @@ -779,9 +760,7 @@ class slots: model_uri=PERSONINFO.has_familial_relationships, domain=None, range=Optional[ - Union[ - Union[dict, FamilialRelationship], List[Union[dict, FamilialRelationship]] - ] + Union[Union[dict, FamilialRelationship], List[Union[dict, FamilialRelationship]]] ], ) @@ -954,9 +933,7 @@ class slots: curie=PERSONINFO.curie("agents"), model_uri=PERSONINFO.agents, domain=None, - range=Optional[ - Union[Dict[Union[str, AgentId], Union[dict, Agent]], List[Union[dict, Agent]]] - ], + range=Optional[Union[Dict[Union[str, AgentId], Union[dict, Agent]], List[Union[dict, Agent]]]], ) slots.organizations = Slot( diff --git a/tests/input/examples/personinfo_basic/model/personinfo_model.py b/tests/input/examples/personinfo_basic/model/personinfo_model.py index 3ef9c0f..7cd8dad 100644 --- a/tests/input/examples/personinfo_basic/model/personinfo_model.py +++ b/tests/input/examples/personinfo_basic/model/personinfo_model.py @@ -13,20 +13,32 @@ from typing import Any, ClassVar, Dict, List, Optional, Union from jsonasobj2 import JsonObj, as_dict -from linkml_runtime.linkml_model.meta import (EnumDefinition, PermissibleValue, - PvFormulaOptions) -from linkml_runtime.linkml_model.types import (Boolean, Date, Float, Integer, - String) +from linkml_runtime.linkml_model.meta import ( + EnumDefinition, + PermissibleValue, + PvFormulaOptions, +) +from linkml_runtime.linkml_model.types import Boolean, Date, Float, Integer, String from linkml_runtime.utils.curienamespace import CurieNamespace -from linkml_runtime.utils.dataclass_extensions_376 import \ - dataclasses_init_fn_with_kwargs +from linkml_runtime.utils.dataclass_extensions_376 import ( + dataclasses_init_fn_with_kwargs, +) from linkml_runtime.utils.enumerations import EnumDefinitionImpl from linkml_runtime.utils.formatutils import camelcase, sfx, underscore -from linkml_runtime.utils.metamodelcore import (Bool, XSDDate, bnode, - empty_dict, empty_list) +from linkml_runtime.utils.metamodelcore import ( + Bool, + XSDDate, + bnode, + empty_dict, + empty_list, +) from linkml_runtime.utils.slot import Slot -from linkml_runtime.utils.yamlutils import (YAMLRoot, extended_float, - extended_int, extended_str) +from linkml_runtime.utils.yamlutils import ( + YAMLRoot, + extended_float, + extended_int, + extended_str, +) from rdflib import Namespace, URIRef metamodel_version = "1.7.0" @@ -43,9 +55,7 @@ ROR = CurieNamespace("ROR", "http://example.org/ror/") FAMREL = CurieNamespace("famrel", "https://example.org/FamilialRelations#") LINKML = CurieNamespace("linkml", "https://w3id.org/linkml/") -PERSONINFO = CurieNamespace( - "personinfo", "https://w3id.org/linkml/examples/personinfo/" -) +PERSONINFO = CurieNamespace("personinfo", "https://w3id.org/linkml/examples/personinfo/") PROV = CurieNamespace("prov", "http://www.w3.org/ns/prov#") RDF = CurieNamespace("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#") RDFS = CurieNamespace("rdfs", "http://www.w3.org/2000/01/rdf-schema#") @@ -174,16 +184,12 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self.gender is not None and not isinstance(self.gender, GenderType): self.gender = GenderType(self.gender) - if self.current_address is not None and not isinstance( - self.current_address, Address - ): + if self.current_address is not None and not isinstance(self.current_address, Address): self.current_address = Address(**as_dict(self.current_address)) if not isinstance(self.has_employment_history, list): self.has_employment_history = ( - [self.has_employment_history] - if self.has_employment_history is not None - else [] + [self.has_employment_history] if self.has_employment_history is not None else [] ) self.has_employment_history = [ v if isinstance(v, EmploymentEvent) else EmploymentEvent(**as_dict(v)) @@ -197,17 +203,13 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): else [] ) self.has_familial_relationships = [ - v - if isinstance(v, FamilialRelationship) - else FamilialRelationship(**as_dict(v)) + v if isinstance(v, FamilialRelationship) else FamilialRelationship(**as_dict(v)) for v in self.has_familial_relationships ] if not isinstance(self.has_medical_history, list): self.has_medical_history = ( - [self.has_medical_history] - if self.has_medical_history is not None - else [] + [self.has_medical_history] if self.has_medical_history is not None else [] ) self.has_medical_history = [ v if isinstance(v, MedicalEvent) else MedicalEvent(**as_dict(v)) @@ -269,17 +271,13 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.id, OrganizationId): self.id = OrganizationId(self.id) - if self.mission_statement is not None and not isinstance( - self.mission_statement, str - ): + if self.mission_statement is not None and not isinstance(self.mission_statement, str): self.mission_statement = str(self.mission_statement) if self.founding_date is not None and not isinstance(self.founding_date, str): self.founding_date = str(self.founding_date) - if self.founding_location is not None and not isinstance( - self.founding_location, PlaceId - ): + if self.founding_location is not None and not isinstance(self.founding_location, PlaceId): self.founding_location = PlaceId(self.founding_location) if not isinstance(self.aliases, list): @@ -359,14 +357,10 @@ class Event(YAMLRoot): is_current: Optional[Union[bool, Bool]] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self.started_at_time is not None and not isinstance( - self.started_at_time, XSDDate - ): + if self.started_at_time is not None and not isinstance(self.started_at_time, XSDDate): self.started_at_time = XSDDate(self.started_at_time) - if self.ended_at_time is not None and not isinstance( - self.ended_at_time, XSDDate - ): + if self.ended_at_time is not None and not isinstance(self.ended_at_time, XSDDate): self.ended_at_time = XSDDate(self.ended_at_time) if self.duration is not None and not isinstance(self.duration, float): @@ -452,14 +446,10 @@ class Relationship(YAMLRoot): related_to: Optional[str] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self.started_at_time is not None and not isinstance( - self.started_at_time, XSDDate - ): + if self.started_at_time is not None and not isinstance(self.started_at_time, XSDDate): self.started_at_time = XSDDate(self.started_at_time) - if self.ended_at_time is not None and not isinstance( - self.ended_at_time, XSDDate - ): + if self.ended_at_time is not None and not isinstance(self.ended_at_time, XSDDate): self.ended_at_time = XSDDate(self.ended_at_time) if self.related_to is not None and not isinstance(self.related_to, str): @@ -507,9 +497,7 @@ class EmploymentEvent(Event): type: Optional[Union[str, "RelationshipType"]] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self.employed_at is not None and not isinstance( - self.employed_at, OrganizationId - ): + if self.employed_at is not None and not isinstance(self.employed_at, OrganizationId): self.employed_at = OrganizationId(self.employed_at) if self.type is not None and not isinstance(self.type, RelationshipType): @@ -536,14 +524,10 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self.in_location is not None and not isinstance(self.in_location, PlaceId): self.in_location = PlaceId(self.in_location) - if self.diagnosis is not None and not isinstance( - self.diagnosis, DiagnosisConcept - ): + if self.diagnosis is not None and not isinstance(self.diagnosis, DiagnosisConcept): self.diagnosis = DiagnosisConcept(**as_dict(self.diagnosis)) - if self.procedure is not None and not isinstance( - self.procedure, ProcedureConcept - ): + if self.procedure is not None and not isinstance(self.procedure, ProcedureConcept): self.procedure = ProcedureConcept(**as_dict(self.procedure)) if self.type is not None and not isinstance(self.type, RelationshipType): @@ -580,9 +564,7 @@ class Container(YAMLRoot): class_model_uri: ClassVar[URIRef] = PERSONINFO.Container persons: Optional[ - Union[ - Dict[Union[str, PersonId], Union[dict, Person]], List[Union[dict, Person]] - ] + Union[Dict[Union[str, PersonId], Union[dict, Person]], List[Union[dict, Person]]] ] = empty_dict() organizations: Optional[ Union[ @@ -758,9 +740,7 @@ class slots: curie=PERSONINFO.curie("has_employment_history"), model_uri=PERSONINFO.has_employment_history, domain=None, - range=Optional[ - Union[Union[dict, EmploymentEvent], List[Union[dict, EmploymentEvent]]] - ], + range=Optional[Union[Union[dict, EmploymentEvent], List[Union[dict, EmploymentEvent]]]], ) slots.has_medical_history = Slot( @@ -779,9 +759,7 @@ class slots: model_uri=PERSONINFO.has_familial_relationships, domain=None, range=Optional[ - Union[ - Union[dict, FamilialRelationship], List[Union[dict, FamilialRelationship]] - ] + Union[Union[dict, FamilialRelationship], List[Union[dict, FamilialRelationship]]] ], ) @@ -936,9 +914,7 @@ class slots: model_uri=PERSONINFO.persons, domain=None, range=Optional[ - Union[ - Dict[Union[str, PersonId], Union[dict, Person]], List[Union[dict, Person]] - ] + Union[Dict[Union[str, PersonId], Union[dict, Person]], List[Union[dict, Person]]] ], ) diff --git a/tests/test_cli/test_cli.py b/tests/test_cli/test_cli.py index 81d72dc..1379ad5 100644 --- a/tests/test_cli/test_cli.py +++ b/tests/test_cli/test_cli.py @@ -7,9 +7,15 @@ from linkml_runtime.loaders import yaml_loader from linkml_transformer.cli.cli import main -from tests import (DENORM_SPECIFICATION, FLATTENING_DATA, NORM_SCHEMA, - PERSONINFO_CONTAINER_DATA, PERSONINFO_SRC_SCHEMA, - PERSONINFO_TGT_SCHEMA, PERSONINFO_TR) +from tests import ( + DENORM_SPECIFICATION, + FLATTENING_DATA, + NORM_SCHEMA, + PERSONINFO_CONTAINER_DATA, + PERSONINFO_SRC_SCHEMA, + PERSONINFO_TGT_SCHEMA, + PERSONINFO_TR, +) class TestCommandLineInterface(unittest.TestCase): diff --git a/tests/test_transformer/test_object_transformer.py b/tests/test_transformer/test_object_transformer.py index 8cd2f75..f13a25f 100644 --- a/tests/test_transformer/test_object_transformer.py +++ b/tests/test_transformer/test_object_transformer.py @@ -3,8 +3,11 @@ import yaml from linkml_runtime import SchemaView -from linkml_runtime.linkml_model import (ClassDefinition, SchemaDefinition, - SlotDefinition) +from linkml_runtime.linkml_model import ( + ClassDefinition, + SchemaDefinition, + SlotDefinition, +) from linkml_runtime.loaders import yaml_loader import tests.input.examples.flattening.model.denormalized_model as sssom_tgt_dm @@ -14,9 +17,16 @@ from linkml_transformer.datamodel.transformer_model import * from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.utils.dynamic_object import dynamic_object -from tests import (DENORM_SCHEMA, DENORM_SPECIFICATION, FLATTENING_DATA, - NORM_SCHEMA, PERSONINFO_DATA, PERSONINFO_SRC_SCHEMA, - PERSONINFO_TGT_SCHEMA, PERSONINFO_TR) +from tests import ( + DENORM_SCHEMA, + DENORM_SPECIFICATION, + FLATTENING_DATA, + NORM_SCHEMA, + PERSONINFO_DATA, + PERSONINFO_SRC_SCHEMA, + PERSONINFO_TGT_SCHEMA, + PERSONINFO_TR, +) AGE_STRING = "33 years" @@ -78,9 +88,7 @@ def test_transform_simple_object(self): # enum derivations are implemented for n in range(len(expected)): fr = obj.has_familial_relationships[n] - self.assertEqual( - src_dm.FamilialRelationshipType(expected[n]["enum"]), fr.type - ) + self.assertEqual(src_dm.FamilialRelationshipType(expected[n]["enum"]), fr.type) target_obj = tr.transform_object(obj, target_class=tgt_dm.Agent) self.assertIsInstance(target_obj, tgt_dm.Agent) @@ -117,9 +125,7 @@ def test_transform_container_object(self): tr = self.tr person = yaml_loader.load(str(PERSONINFO_DATA), target_class=src_dm.Person) obj = src_dm.Container(persons=[person]) - target_obj: tgt_dm.Container = tr.transform_object( - obj, target_class=tgt_dm.Container - ) + target_obj: tgt_dm.Container = tr.transform_object(obj, target_class=tgt_dm.Container) self.assertIsInstance(target_obj, tgt_dm.Container) agents = target_obj.agents agent = agents[0] @@ -209,9 +215,7 @@ def test_denormalized_transform_dict(self): self.assertEqual( mset["mappings"], [{"subject": "X:1", "object": "Y:1", "predicate": "P:1"}] ) - self.assertEqual( - mset["entities"], {"X:1": {"name": "x1"}, "Y:1": {"name": "y1"}} - ) + self.assertEqual(mset["entities"], {"X:1": {"name": "x1"}, "Y:1": {"name": "y1"}}) tr.index(mset, "MappingSet") target_obj = tr.transform(mset, source_type="MappingSet") assert type(target_obj) == dict @@ -281,9 +285,7 @@ def test_cardinalities(self): class_name = "MyClass" att_name = "my_att" val = "v1" - for source_multivalued, target_multivalued, explicit in itertools.product( - tf, tf, tf - ): + for source_multivalued, target_multivalued, explicit in itertools.product(tf, tf, tf): def mk(mv: bool, explicit: bool = False): cls = ClassDefinition(class_name) @@ -316,9 +318,7 @@ def mk(mv: bool, explicit: bool = False): target_schemaview=SchemaView(target_schema), ) target_instance = tr.transform(source_instance, class_name) - self.assertEqual( - [val] if target_multivalued else val, target_instance[att_name] - ) + self.assertEqual([val] if target_multivalued else val, target_instance[att_name]) if __name__ == "__main__": diff --git a/tests/test_transformer/test_transformer_examples.py b/tests/test_transformer/test_transformer_examples.py index 8719ad3..148fc76 100644 --- a/tests/test_transformer/test_transformer_examples.py +++ b/tests/test_transformer/test_transformer_examples.py @@ -3,8 +3,7 @@ import yaml from linkml_transformer.transformer.object_transformer import ObjectTransformer -from linkml_transformer.utils.multi_file_transformer import \ - MultiFileTransformer +from linkml_transformer.utils.multi_file_transformer import MultiFileTransformer from tests import EXAMPLE_DIR EXAMPLE_PROJECTS = [ diff --git a/tests/test_utils/test_dynamic_object.py b/tests/test_utils/test_dynamic_object.py index dc140e5..b1a1771 100644 --- a/tests/test_utils/test_dynamic_object.py +++ b/tests/test_utils/test_dynamic_object.py @@ -7,9 +7,16 @@ from linkml_transformer.datamodel.transformer_model import * from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.utils.dynamic_object import dynamic_object -from tests import (DENORM_SCHEMA, DENORM_SPECIFICATION, FLATTENING_DATA, - NORM_SCHEMA, PERSONINFO_DATA, PERSONINFO_SRC_SCHEMA, - PERSONINFO_TGT_SCHEMA, PERSONINFO_TR) +from tests import ( + DENORM_SCHEMA, + DENORM_SPECIFICATION, + FLATTENING_DATA, + NORM_SCHEMA, + PERSONINFO_DATA, + PERSONINFO_SRC_SCHEMA, + PERSONINFO_TGT_SCHEMA, + PERSONINFO_TR, +) class DynamicObjectTestCase(unittest.TestCase): From 1cd67d215181d756fdce3f88b0f1d204a9be50f6 Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 2 May 2023 11:20:14 -0700 Subject: [PATCH 11/15] using latest referencevalidator --- poetry.lock | 24 ++++++++++++------------ pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2d14270..ff0f5d2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -1176,18 +1176,18 @@ files = [ [package.dependencies] attrs = ">=17.4.0" -fqdn = {version = "*", optional = true, markers = "extra == \"format\""} -idna = {version = "*", optional = true, markers = "extra == \"format\""} +fqdn = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +idna = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} -isoduration = {version = "*", optional = true, markers = "extra == \"format\""} -jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format\""} +isoduration = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format-nongpl\""} pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" -rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format\""} +rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} rfc3986-validator = {version = ">0.1.0", optional = true, markers = "extra == \"format-nongpl\""} rfc3987 = {version = "*", optional = true, markers = "extra == \"format\""} -uri-template = {version = "*", optional = true, markers = "extra == \"format\""} -webcolors = {version = ">=1.11", optional = true, markers = "extra == \"format\""} +uri-template = {version = "*", optional = true, markers = "extra == \"format-nongpl\""} +webcolors = {version = ">=1.11", optional = true, markers = "extra == \"format-nongpl\""} [package.extras] format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] @@ -1455,14 +1455,14 @@ linkml-runtime = ">=1.1.6" [[package]] name = "linkml-runtime" -version = "1.4.8" +version = "1.5.1" description = "Runtime environment for LinkML, the Linked open data modeling language" category = "main" optional = false python-versions = ">=3.7.6,<4.0.0" files = [ - {file = "linkml_runtime-1.4.8-py3-none-any.whl", hash = "sha256:2a48767c0e0a8d82d2483bd46d8218ec7a3afc8d54135999ef560be60153cbcd"}, - {file = "linkml_runtime-1.4.8.tar.gz", hash = "sha256:7ead386bdc71d61b512650ab81d5f505b28010a0fdd6e0f3a4f10dc5019b385a"}, + {file = "linkml_runtime-1.5.1-py3-none-any.whl", hash = "sha256:1ae23259500d7fde465313547d8be33049fc071857e6c546b5212b23a9bf0016"}, + {file = "linkml_runtime-1.5.1.tar.gz", hash = "sha256:fd5cdd6403ae5e087e1e5e4023542a6da6e6df4538e9b516c30d6315f8be61dc"}, ] [package.dependencies] @@ -3877,4 +3877,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "dbf5de14a44096ca4090d8a64660dd96858c6cf6ddc4750ef8325dd1e0889731" +content-hash = "2f8318cddb8c68c066d139c207e0b79be3b3ea1ee42a0c7fff8e74b33e1edafa" diff --git a/pyproject.toml b/pyproject.toml index cb8f3dc..e3b9aca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.8" -linkml-runtime = "^1.4.5" +linkml-runtime = "^1.5.1" [tool.poetry.dev-dependencies] pytest = "^5.2" From fb97d5ab5b7fa4000a5190309cb38e365946c3c8 Mon Sep 17 00:00:00 2001 From: cmungall Date: Tue, 2 May 2023 13:07:16 -0700 Subject: [PATCH 12/15] upgrading to latest --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index ff0f5d2..c8af29d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1455,14 +1455,14 @@ linkml-runtime = ">=1.1.6" [[package]] name = "linkml-runtime" -version = "1.5.1" +version = "1.5.2" description = "Runtime environment for LinkML, the Linked open data modeling language" category = "main" optional = false python-versions = ">=3.7.6,<4.0.0" files = [ - {file = "linkml_runtime-1.5.1-py3-none-any.whl", hash = "sha256:1ae23259500d7fde465313547d8be33049fc071857e6c546b5212b23a9bf0016"}, - {file = "linkml_runtime-1.5.1.tar.gz", hash = "sha256:fd5cdd6403ae5e087e1e5e4023542a6da6e6df4538e9b516c30d6315f8be61dc"}, + {file = "linkml_runtime-1.5.2-py3-none-any.whl", hash = "sha256:33c8ac80a9fc92e26298c7d060866e19b9d186f1250a0d880ac431034a40a2fa"}, + {file = "linkml_runtime-1.5.2.tar.gz", hash = "sha256:22c463f090db0ec37d46eac0d68b78323dee5de1f4856c3be9e807125796da51"}, ] [package.dependencies] @@ -3877,4 +3877,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "2f8318cddb8c68c066d139c207e0b79be3b3ea1ee42a0c7fff8e74b33e1edafa" +content-hash = "54c6367b27fec1069f96e57a17443fb6dbe0f9e74d6f7146a851cf838ce3789a" diff --git a/pyproject.toml b/pyproject.toml index e3b9aca..26e4332 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.8" -linkml-runtime = "^1.5.1" +linkml-runtime = "^1.5.2" [tool.poetry.dev-dependencies] pytest = "^5.2" From a614277b4f4ee239dfffadeed71f9d601dfb34ba Mon Sep 17 00:00:00 2001 From: cmungall Date: Wed, 3 May 2023 16:13:17 -0700 Subject: [PATCH 13/15] lint --- tests/test_datamodel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_datamodel.py b/tests/test_datamodel.py index be203fe..9836fc9 100644 --- a/tests/test_datamodel.py +++ b/tests/test_datamodel.py @@ -3,9 +3,8 @@ from linkml_runtime.dumpers import yaml_dumper from linkml_runtime.loaders import yaml_loader -from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.datamodel.transformer_model import TransformationSpecification - +from linkml_transformer.transformer.object_transformer import ObjectTransformer from tests import PERSONINFO_TR From a233f588d33262aad44c10bfb8ff8ff4057a1771 Mon Sep 17 00:00:00 2001 From: cmungall Date: Wed, 3 May 2023 16:21:43 -0700 Subject: [PATCH 14/15] lint --- src/linkml_transformer/cli/cli.py | 2 -- src/linkml_transformer/compiler/awk_compiler.py | 2 -- src/linkml_transformer/compiler/compiler.py | 3 +-- src/linkml_transformer/compiler/python_compiler.py | 7 +++---- src/linkml_transformer/compiler/r2rml_compiler.py | 2 -- src/linkml_transformer/compiler/sparql_compiler.py | 2 -- src/linkml_transformer/compiler/sql_compiler.py | 2 -- src/linkml_transformer/compiler/sssom_compiler.py | 2 -- src/linkml_transformer/importer/importer.py | 3 --- .../utils/multi_file_transformer.py | 11 ++--------- tests/test_cli/test_cli.py | 1 - tests/test_compiler/test_python_compiler.py | 5 ----- tests/test_datamodel.py | 4 ---- tests/test_mapper/test_schema_mapper.py | 1 - tests/test_transformer/test_transformer_examples.py | 3 --- tests/test_utils/test_dynamic_object.py | 11 +---------- 16 files changed, 7 insertions(+), 54 deletions(-) diff --git a/src/linkml_transformer/cli/cli.py b/src/linkml_transformer/cli/cli.py index e845b2f..2d629be 100644 --- a/src/linkml_transformer/cli/cli.py +++ b/src/linkml_transformer/cli/cli.py @@ -11,9 +11,7 @@ import yaml from linkml_runtime import SchemaView from linkml_runtime.dumpers import yaml_dumper -from linkml_runtime.loaders import yaml_loader -from linkml_transformer.datamodel.transformer_model import TransformationSpecification from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper from linkml_transformer.transformer.object_transformer import ObjectTransformer diff --git a/src/linkml_transformer/compiler/awk_compiler.py b/src/linkml_transformer/compiler/awk_compiler.py index 9bb11be..3cc6864 100644 --- a/src/linkml_transformer/compiler/awk_compiler.py +++ b/src/linkml_transformer/compiler/awk_compiler.py @@ -1,5 +1,3 @@ -from linkml_runtime.utils.yamlutils import YAMLRoot - from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler from linkml_transformer.datamodel.transformer_model import TransformationSpecification diff --git a/src/linkml_transformer/compiler/compiler.py b/src/linkml_transformer/compiler/compiler.py index 2f46079..f2dd507 100644 --- a/src/linkml_transformer/compiler/compiler.py +++ b/src/linkml_transformer/compiler/compiler.py @@ -1,9 +1,8 @@ -from abc import ABC, abstractmethod +from abc import ABC from dataclasses import dataclass from typing import Iterator from linkml_runtime import SchemaView -from linkml_runtime.utils.yamlutils import YAMLRoot from linkml_transformer.datamodel.transformer_model import TransformationSpecification diff --git a/src/linkml_transformer/compiler/python_compiler.py b/src/linkml_transformer/compiler/python_compiler.py index 528fec8..1b07ffd 100644 --- a/src/linkml_transformer/compiler/python_compiler.py +++ b/src/linkml_transformer/compiler/python_compiler.py @@ -2,9 +2,8 @@ from typing import Iterator from jinja2 import Template -from linkml_runtime.utils.yamlutils import YAMLRoot -from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler +from linkml_transformer.compiler.compiler import Compiler from linkml_transformer.datamodel.transformer_model import ( ClassDerivation, TransformationSpecification, @@ -30,14 +29,14 @@ {%- else -%} None {%- endif -%} -{%- endif -%} +{%- endif -%} {%- endmacro %} def derive_{{ cd.name }}( source_object: {{ source_module }}.{{ cd.populated_from }} ) -> {{ target_module }}.{{ cd.name }}: return {{ cd.populated_from }}( {%- for sd in cd.slot_derivations.values() %} - {{ sd.name }}={{ gen_slot_derivation(sd) }}, + {{ sd.name }}={{ gen_slot_derivation(sd) }}, {%- endfor %} ) diff --git a/src/linkml_transformer/compiler/r2rml_compiler.py b/src/linkml_transformer/compiler/r2rml_compiler.py index 6f5a889..b78b70a 100644 --- a/src/linkml_transformer/compiler/r2rml_compiler.py +++ b/src/linkml_transformer/compiler/r2rml_compiler.py @@ -1,5 +1,3 @@ -from linkml_runtime.utils.yamlutils import YAMLRoot - from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler from linkml_transformer.datamodel.transformer_model import TransformationSpecification diff --git a/src/linkml_transformer/compiler/sparql_compiler.py b/src/linkml_transformer/compiler/sparql_compiler.py index 4799754..7ac5b89 100644 --- a/src/linkml_transformer/compiler/sparql_compiler.py +++ b/src/linkml_transformer/compiler/sparql_compiler.py @@ -1,5 +1,3 @@ -from linkml_runtime.utils.yamlutils import YAMLRoot - from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler from linkml_transformer.datamodel.transformer_model import TransformationSpecification diff --git a/src/linkml_transformer/compiler/sql_compiler.py b/src/linkml_transformer/compiler/sql_compiler.py index 5d8f0e7..2fab732 100644 --- a/src/linkml_transformer/compiler/sql_compiler.py +++ b/src/linkml_transformer/compiler/sql_compiler.py @@ -1,5 +1,3 @@ -from linkml_runtime.utils.yamlutils import YAMLRoot - from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler from linkml_transformer.datamodel.transformer_model import TransformationSpecification diff --git a/src/linkml_transformer/compiler/sssom_compiler.py b/src/linkml_transformer/compiler/sssom_compiler.py index fd58531..d94bc7b 100644 --- a/src/linkml_transformer/compiler/sssom_compiler.py +++ b/src/linkml_transformer/compiler/sssom_compiler.py @@ -1,5 +1,3 @@ -from linkml_runtime.utils.yamlutils import YAMLRoot - from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler from linkml_transformer.datamodel.transformer_model import TransformationSpecification diff --git a/src/linkml_transformer/importer/importer.py b/src/linkml_transformer/importer/importer.py index 49825ea..2c7ef29 100644 --- a/src/linkml_transformer/importer/importer.py +++ b/src/linkml_transformer/importer/importer.py @@ -2,9 +2,6 @@ from dataclasses import dataclass from typing import Any -from linkml_runtime import SchemaView -from linkml_runtime.utils.yamlutils import YAMLRoot - from linkml_transformer.datamodel.transformer_model import TransformationSpecification diff --git a/src/linkml_transformer/utils/multi_file_transformer.py b/src/linkml_transformer/utils/multi_file_transformer.py index 89ce93c..3564bc1 100644 --- a/src/linkml_transformer/utils/multi_file_transformer.py +++ b/src/linkml_transformer/utils/multi_file_transformer.py @@ -2,23 +2,16 @@ """ import glob -import json import logging import os import re -import sys from dataclasses import dataclass, field -from io import StringIO from pathlib import Path -from types import ModuleType -from typing import Any, List, Mapping, Optional, TextIO, Union +from typing import List, Mapping, Optional, Union -import click import yaml from linkml_runtime import SchemaView -from linkml_runtime.dumpers import json_dumper, rdflib_dumper, yaml_dumper -from linkml_runtime.linkml_model import ElementName -from linkml_runtime.utils.formatutils import camelcase +from linkml_runtime.dumpers import yaml_dumper from pydantic import BaseModel from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper diff --git a/tests/test_cli/test_cli.py b/tests/test_cli/test_cli.py index 9b4ca9d..a5cb5e7 100644 --- a/tests/test_cli/test_cli.py +++ b/tests/test_cli/test_cli.py @@ -12,7 +12,6 @@ NORM_SCHEMA, PERSONINFO_CONTAINER_DATA, PERSONINFO_SRC_SCHEMA, - PERSONINFO_TGT_SCHEMA, PERSONINFO_TR, ) diff --git a/tests/test_compiler/test_python_compiler.py b/tests/test_compiler/test_python_compiler.py index b973fb5..abe30c7 100644 --- a/tests/test_compiler/test_python_compiler.py +++ b/tests/test_compiler/test_python_compiler.py @@ -1,13 +1,8 @@ import unittest from linkml_runtime import SchemaView -from linkml_runtime.dumpers import yaml_dumper -from linkml_runtime.loaders import yaml_loader from linkml_transformer.compiler.python_compiler import PythonCompiler -from linkml_transformer.datamodel.transformer_model import * -from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper -from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.utils.loaders import load_specification from tests import SCHEMA1, SPECIFICATION diff --git a/tests/test_datamodel.py b/tests/test_datamodel.py index 9836fc9..ff2ca0a 100644 --- a/tests/test_datamodel.py +++ b/tests/test_datamodel.py @@ -1,9 +1,5 @@ import unittest -from linkml_runtime.dumpers import yaml_dumper -from linkml_runtime.loaders import yaml_loader - -from linkml_transformer.datamodel.transformer_model import TransformationSpecification from linkml_transformer.transformer.object_transformer import ObjectTransformer from tests import PERSONINFO_TR diff --git a/tests/test_mapper/test_schema_mapper.py b/tests/test_mapper/test_schema_mapper.py index 211671b..4fb51a5 100644 --- a/tests/test_mapper/test_schema_mapper.py +++ b/tests/test_mapper/test_schema_mapper.py @@ -2,7 +2,6 @@ from linkml_runtime import SchemaView from linkml_runtime.dumpers import yaml_dumper -from linkml_runtime.loaders import yaml_loader from linkml_transformer.datamodel.transformer_model import ( ClassDerivation, diff --git a/tests/test_transformer/test_transformer_examples.py b/tests/test_transformer/test_transformer_examples.py index 148fc76..54bb04e 100644 --- a/tests/test_transformer/test_transformer_examples.py +++ b/tests/test_transformer/test_transformer_examples.py @@ -1,8 +1,5 @@ import unittest -import yaml - -from linkml_transformer.transformer.object_transformer import ObjectTransformer from linkml_transformer.utils.multi_file_transformer import MultiFileTransformer from tests import EXAMPLE_DIR diff --git a/tests/test_utils/test_dynamic_object.py b/tests/test_utils/test_dynamic_object.py index 94760d8..3ddc277 100644 --- a/tests/test_utils/test_dynamic_object.py +++ b/tests/test_utils/test_dynamic_object.py @@ -4,16 +4,7 @@ from linkml_runtime import SchemaView from linkml_transformer.utils.dynamic_object import dynamic_object -from tests import ( - DENORM_SCHEMA, - DENORM_SPECIFICATION, - FLATTENING_DATA, - NORM_SCHEMA, - PERSONINFO_DATA, - PERSONINFO_SRC_SCHEMA, - PERSONINFO_TGT_SCHEMA, - PERSONINFO_TR, -) +from tests import FLATTENING_DATA, NORM_SCHEMA class DynamicObjectTestCase(unittest.TestCase): From 9e4b0ec141f024f1be3fa9a0809f031a5869ca46 Mon Sep 17 00:00:00 2001 From: cmungall Date: Wed, 3 May 2023 16:24:39 -0700 Subject: [PATCH 15/15] dependencies --- poetry.lock | 133 ++++++++++++++++++++++++++++++++++--------------- pyproject.toml | 4 +- 2 files changed, 96 insertions(+), 41 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9d72362..fa55394 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1729,14 +1729,14 @@ files = [ [[package]] name = "mkdocs" -version = "1.4.2" +version = "1.4.3" description = "Project documentation with Markdown." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "mkdocs-1.4.2-py3-none-any.whl", hash = "sha256:c8856a832c1e56702577023cd64cc5f84948280c1c0fcc6af4cd39006ea6aa8c"}, - {file = "mkdocs-1.4.2.tar.gz", hash = "sha256:8947af423a6d0facf41ea1195b8e1e8c85ad94ac95ae307fe11232e0424b11c5"}, + {file = "mkdocs-1.4.3-py3-none-any.whl", hash = "sha256:6ee46d309bda331aac915cd24aab882c179a933bd9e77b80ce7d2eaaa3f689dd"}, + {file = "mkdocs-1.4.3.tar.gz", hash = "sha256:5955093bbd4dd2e9403c5afaf57324ad8b04f16886512a3ee6ef828956481c57"}, ] [package.dependencies] @@ -2922,45 +2922,100 @@ rdflib-jsonld = "0.6.1" [[package]] name = "regex" -version = "2023.5.2" +version = "2023.5.5" description = "Alternative regular expression module, to replace re." category = "dev" optional = false -python-versions = ">=3.8" +python-versions = ">=3.6" files = [ - {file = "regex-2023.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c801b2b2326eb645a0f3cb00c23bbae422c1efb8a887120d93a9d4b313252728"}, - {file = "regex-2023.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5cd5f25d9fef92fec3a3df0e10260c76519874400c5efca19994d8c6402e6ef"}, - {file = "regex-2023.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72df39b0389849160f30ea49d16f6ffead6e6bc8e4145471cbe5422d1e074c82"}, - {file = "regex-2023.5.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8fbc1a856bbab364baa7bc6c27dce5f5e6939ff4fc2b54461311111249b6767"}, - {file = "regex-2023.5.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bb848ee40dc625a661247c884cc816fb285a265bb08a76cad2d65c67bb37a815"}, - {file = "regex-2023.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:364dcc8f95543ba37456b5ad765d13156f51c1ad616565b025f78bb6e5bc4823"}, - {file = "regex-2023.5.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27dd7deb985b5f6bf21021f9be2f9c23f775923ac6be4193dfc42d0940b078fd"}, - {file = "regex-2023.5.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9b1b82baa40b01239b538e49f8240b67bd41912f556b3dbb9bdfdb617cdb1ce0"}, - {file = "regex-2023.5.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1a4f57c4526197bf671e6615c5a364dc1738eae4d2199e4c4b94299f16ba1917"}, - {file = "regex-2023.5.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:76ebbc1a47740dd3fb0fab3e05bdd79705fbe92bcb28d1b509c59fe8fb3405ee"}, - {file = "regex-2023.5.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bd32760811428cd069e6849ab03ba12c0deea4b579804f866981db283325ea6e"}, - {file = "regex-2023.5.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:f37fdc38e63df6cac2565adba4a8a54d097ccb77f59227a5aeb78ebca1ca4610"}, - {file = "regex-2023.5.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6a25a37baa85c72ecb50976a0c0c0c437f150e62d37b910db63ff049f33ac375"}, - {file = "regex-2023.5.2-cp310-cp310-win32.whl", hash = "sha256:ee2bc5df35672b36c847143d772993380d325187510d1ff5d7dba7a65ad56b62"}, - {file = "regex-2023.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:66b1c91073642853c1fe3d27c3f797d3bfd0b1b9cbb567fb21428950f6b9221f"}, - {file = "regex-2023.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c2e55a91c2a104e14b799a6802b574c24b3e8b70d5f3a47f8025ea38aade2d2a"}, - {file = "regex-2023.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30b37d9b8598dee35df0546d7648c2a368a3ad668e2e0c9c33f549b01666b456"}, - {file = "regex-2023.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bf3cbf3d9386885e56e51d9cde879611a0e365024a61638ec072a5c9d227cc9"}, - {file = "regex-2023.5.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9b643701f0998961f9847c5f12714517ba321f1af3a646d8a4c1a0f4e8880793"}, - {file = "regex-2023.5.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48eb4daf4706d16c4c854946f0b9dc2b1f52a5e9cccdf27a3b2cb1f44cb80b8d"}, - {file = "regex-2023.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8ab95916ab1bc88e9cdfd2de585401d2764d4c60805eeb2aae38be7adf07edf"}, - {file = "regex-2023.5.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6b1397cd6094a0e8fd8e7cd986f3ad27bcdb2622ce7b11b626dc1491b9a7ccef"}, - {file = "regex-2023.5.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:68935a664cdbf0cc54eec8a1df35a952f95cb4d58d6f84a061fef061084847f8"}, - {file = "regex-2023.5.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bbb5cb89ea732a19b41ece443199bf91f9984f3ec9f847e27dd9227ecab8e86e"}, - {file = "regex-2023.5.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:062b7fa469e90673d0845f6dab0bbbd8d602a7db18c4842d5ee4b13829219cbc"}, - {file = "regex-2023.5.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b4a51fbdddb6fbb21c40cc0c646912f310dbe73deef0f07c7c471bd1dcfe0c99"}, - {file = "regex-2023.5.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bed5a1af5ffd25bce1244ddef9959b3b2fbc49d4f7493213fe7d979bf42407c"}, - {file = "regex-2023.5.2-cp311-cp311-win32.whl", hash = "sha256:aec0255d348c5e6bb0d475bcb5af4c3483deb4a9839a13906d4a01f5b4af70a8"}, - {file = "regex-2023.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:f2ffde6bd983564609547ad695803f61785b24ac255549115a8349135562c6e0"}, - {file = "regex-2023.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:343771b965923a505a937e7e5c0cc047f303771f119e75025e66d3615eedbcf3"}, - {file = "regex-2023.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3817ef2ff8f2b79832adb75557f76b19c973c9852505a3feabdb0638a78c3e57"}, - {file = "regex-2023.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30931415755e549785808ba9cb1019dabefe1f387cf46e5d1a5fa475832784c0"}, - {file = "regex-2023.5.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ed3fec994eebd1323ed40afb06bed40bdf84d0f10357056afd12b862f7487dd"}, + {file = "regex-2023.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:48c9ec56579d4ba1c88f42302194b8ae2350265cb60c64b7b9a88dcb7fbde309"}, + {file = "regex-2023.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f4541550459c08fdd6f97aa4e24c6f1932eec780d58a2faa2068253df7d6ff"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53e22e4460f0245b468ee645156a4f84d0fc35a12d9ba79bd7d79bdcd2f9629d"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b870b6f632fc74941cadc2a0f3064ed8409e6f8ee226cdfd2a85ae50473aa94"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:171c52e320fe29260da550d81c6b99f6f8402450dc7777ef5ced2e848f3b6f8f"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad5524c2aedaf9aa14ef1bc9327f8abd915699dea457d339bebbe2f0d218f86"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a0f874ee8c0bc820e649c900243c6d1e6dc435b81da1492046716f14f1a2a96"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e645c757183ee0e13f0bbe56508598e2d9cd42b8abc6c0599d53b0d0b8dd1479"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a4c5da39bca4f7979eefcbb36efea04471cd68db2d38fcbb4ee2c6d440699833"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5e3f4468b8c6fd2fd33c218bbd0a1559e6a6fcf185af8bb0cc43f3b5bfb7d636"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:59e4b729eae1a0919f9e4c0fc635fbcc9db59c74ad98d684f4877be3d2607dd6"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ba73a14e9c8f9ac409863543cde3290dba39098fc261f717dc337ea72d3ebad2"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0bbd5dcb19603ab8d2781fac60114fb89aee8494f4505ae7ad141a3314abb1f9"}, + {file = "regex-2023.5.5-cp310-cp310-win32.whl", hash = "sha256:40005cbd383438aecf715a7b47fe1e3dcbc889a36461ed416bdec07e0ef1db66"}, + {file = "regex-2023.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:59597cd6315d3439ed4b074febe84a439c33928dd34396941b4d377692eca810"}, + {file = "regex-2023.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f08276466fedb9e36e5193a96cb944928301152879ec20c2d723d1031cd4ddd"}, + {file = "regex-2023.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cd46f30e758629c3ee91713529cfbe107ac50d27110fdcc326a42ce2acf4dafc"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2910502f718828cecc8beff004917dcf577fc5f8f5dd40ffb1ea7612124547b"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:445d6f4fc3bd9fc2bf0416164454f90acab8858cd5a041403d7a11e3356980e8"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18196c16a584619c7c1d843497c069955d7629ad4a3fdee240eb347f4a2c9dbe"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33d430a23b661629661f1fe8395be2004006bc792bb9fc7c53911d661b69dd7e"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72a28979cc667e5f82ef433db009184e7ac277844eea0f7f4d254b789517941d"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f764e4dfafa288e2eba21231f455d209f4709436baeebb05bdecfb5d8ddc3d35"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23d86ad2121b3c4fc78c58f95e19173790e22ac05996df69b84e12da5816cb17"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:690a17db524ee6ac4a27efc5406530dd90e7a7a69d8360235323d0e5dafb8f5b"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:1ecf3dcff71f0c0fe3e555201cbe749fa66aae8d18f80d2cc4de8e66df37390a"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:811040d7f3dd9c55eb0d8b00b5dcb7fd9ae1761c454f444fd9f37fe5ec57143a"}, + {file = "regex-2023.5.5-cp311-cp311-win32.whl", hash = "sha256:c8c143a65ce3ca42e54d8e6fcaf465b6b672ed1c6c90022794a802fb93105d22"}, + {file = "regex-2023.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:586a011f77f8a2da4b888774174cd266e69e917a67ba072c7fc0e91878178a80"}, + {file = "regex-2023.5.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b6365703e8cf1644b82104cdd05270d1a9f043119a168d66c55684b1b557d008"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a56c18f21ac98209da9c54ae3ebb3b6f6e772038681d6cb43b8d53da3b09ee81"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8b942d8b3ce765dbc3b1dad0a944712a89b5de290ce8f72681e22b3c55f3cc8"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:844671c9c1150fcdac46d43198364034b961bd520f2c4fdaabfc7c7d7138a2dd"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2ce65bdeaf0a386bb3b533a28de3994e8e13b464ac15e1e67e4603dd88787fa"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fee0016cc35a8a91e8cc9312ab26a6fe638d484131a7afa79e1ce6165328a135"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:18f05d14f14a812fe9723f13afafefe6b74ca042d99f8884e62dbd34dcccf3e2"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:941b3f1b2392f0bcd6abf1bc7a322787d6db4e7457be6d1ffd3a693426a755f2"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:921473a93bcea4d00295799ab929522fc650e85c6b9f27ae1e6bb32a790ea7d3"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:e2205a81f815b5bb17e46e74cc946c575b484e5f0acfcb805fb252d67e22938d"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:385992d5ecf1a93cb85adff2f73e0402dd9ac29b71b7006d342cc920816e6f32"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:890a09cb0a62198bff92eda98b2b507305dd3abf974778bae3287f98b48907d3"}, + {file = "regex-2023.5.5-cp36-cp36m-win32.whl", hash = "sha256:821a88b878b6589c5068f4cc2cfeb2c64e343a196bc9d7ac68ea8c2a776acd46"}, + {file = "regex-2023.5.5-cp36-cp36m-win_amd64.whl", hash = "sha256:7918a1b83dd70dc04ab5ed24c78ae833ae8ea228cef84e08597c408286edc926"}, + {file = "regex-2023.5.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:338994d3d4ca4cf12f09822e025731a5bdd3a37aaa571fa52659e85ca793fb67"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a69cf0c00c4d4a929c6c7717fd918414cab0d6132a49a6d8fc3ded1988ed2ea"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f5e06df94fff8c4c85f98c6487f6636848e1dc85ce17ab7d1931df4a081f657"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8906669b03c63266b6a7693d1f487b02647beb12adea20f8840c1a087e2dfb5"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fda3e50abad8d0f48df621cf75adc73c63f7243cbe0e3b2171392b445401550"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ac2b7d341dc1bd102be849d6dd33b09701223a851105b2754339e390be0627a"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fb2b495dd94b02de8215625948132cc2ea360ae84fe6634cd19b6567709c8ae2"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aa7d032c1d84726aa9edeb6accf079b4caa87151ca9fabacef31fa028186c66d"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3d45864693351c15531f7e76f545ec35000d50848daa833cead96edae1665559"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21e90a288e6ba4bf44c25c6a946cb9b0f00b73044d74308b5e0afd190338297c"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:10250a093741ec7bf74bcd2039e697f519b028518f605ff2aa7ac1e9c9f97423"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6b8d0c153f07a953636b9cdb3011b733cadd4178123ef728ccc4d5969e67f3c2"}, + {file = "regex-2023.5.5-cp37-cp37m-win32.whl", hash = "sha256:10374c84ee58c44575b667310d5bbfa89fb2e64e52349720a0182c0017512f6c"}, + {file = "regex-2023.5.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9b320677521aabf666cdd6e99baee4fb5ac3996349c3b7f8e7c4eee1c00dfe3a"}, + {file = "regex-2023.5.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:afb1c70ec1e594a547f38ad6bf5e3d60304ce7539e677c1429eebab115bce56e"}, + {file = "regex-2023.5.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cf123225945aa58b3057d0fba67e8061c62d14cc8a4202630f8057df70189051"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a99757ad7fe5c8a2bb44829fc57ced11253e10f462233c1255fe03888e06bc19"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a623564d810e7a953ff1357f7799c14bc9beeab699aacc8b7ab7822da1e952b8"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ced02e3bd55e16e89c08bbc8128cff0884d96e7f7a5633d3dc366b6d95fcd1d6"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1cbe6b5be3b9b698d8cc4ee4dee7e017ad655e83361cd0ea8e653d65e469468"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a6e4b0e0531223f53bad07ddf733af490ba2b8367f62342b92b39b29f72735a"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e9c4f778514a560a9c9aa8e5538bee759b55f6c1dcd35613ad72523fd9175b8"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:256f7f4c6ba145f62f7a441a003c94b8b1af78cee2cccacfc1e835f93bc09426"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd7b68fd2e79d59d86dcbc1ccd6e2ca09c505343445daaa4e07f43c8a9cc34da"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4a5059bd585e9e9504ef9c07e4bc15b0a621ba20504388875d66b8b30a5c4d18"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:6893544e06bae009916a5658ce7207e26ed17385149f35a3125f5259951f1bbe"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c64d5abe91a3dfe5ff250c6bb267ef00dbc01501518225b45a5f9def458f31fb"}, + {file = "regex-2023.5.5-cp38-cp38-win32.whl", hash = "sha256:7923470d6056a9590247ff729c05e8e0f06bbd4efa6569c916943cb2d9b68b91"}, + {file = "regex-2023.5.5-cp38-cp38-win_amd64.whl", hash = "sha256:4035d6945cb961c90c3e1c1ca2feb526175bcfed44dfb1cc77db4fdced060d3e"}, + {file = "regex-2023.5.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:50fd2d9b36938d4dcecbd684777dd12a407add4f9f934f235c66372e630772b0"}, + {file = "regex-2023.5.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d19e57f888b00cd04fc38f5e18d0efbd91ccba2d45039453ab2236e6eec48d4d"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd966475e963122ee0a7118ec9024388c602d12ac72860f6eea119a3928be053"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db09e6c18977a33fea26fe67b7a842f706c67cf8bda1450974d0ae0dd63570df"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6164d4e2a82f9ebd7752a06bd6c504791bedc6418c0196cd0a23afb7f3e12b2d"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84397d3f750d153ebd7f958efaa92b45fea170200e2df5e0e1fd4d85b7e3f58a"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c3efee9bb53cbe7b285760c81f28ac80dc15fa48b5fe7e58b52752e642553f1"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:144b5b017646b5a9392a5554a1e5db0000ae637be4971c9747566775fc96e1b2"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1189fbbb21e2c117fda5303653b61905aeeeea23de4a94d400b0487eb16d2d60"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f83fe9e10f9d0b6cf580564d4d23845b9d692e4c91bd8be57733958e4c602956"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:72aa4746993a28c841e05889f3f1b1e5d14df8d3daa157d6001a34c98102b393"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:de2f780c3242ea114dd01f84848655356af4dd561501896c751d7b885ea6d3a1"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:290fd35219486dfbc00b0de72f455ecdd63e59b528991a6aec9fdfc0ce85672e"}, + {file = "regex-2023.5.5-cp39-cp39-win32.whl", hash = "sha256:732176f5427e72fa2325b05c58ad0b45af341c459910d766f814b0584ac1f9ac"}, + {file = "regex-2023.5.5-cp39-cp39-win_amd64.whl", hash = "sha256:1307aa4daa1cbb23823d8238e1f61292fd07e4e5d8d38a6efff00b67a7cdb764"}, + {file = "regex-2023.5.5.tar.gz", hash = "sha256:7d76a8a1fc9da08296462a18f16620ba73bcbf5909e42383b253ef34d9d5141e"}, ] [[package]] @@ -3910,4 +3965,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "54c6367b27fec1069f96e57a17443fb6dbe0f9e74d6f7146a851cf838ce3789a" +content-hash = "b35ac0bdf7cdbb9385b599cd65a32b275ce66c39f4a1fd2c2a1295c7debdf058" diff --git a/pyproject.toml b/pyproject.toml index 0444523..95aeb88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,12 +7,12 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.8" -linkml-runtime = "^1.5.2" +linkml-runtime = ">=1.5.2" [tool.poetry.dev-dependencies] pytest = "^7.3.1" pytest-cov = "^4.0.0" -linkml = "^1.3.7" +linkml = ">=1.5.0" mkdocs-mermaid2-plugin = "^0.6.0" [tool.poetry.scripts]