WIP Use entity rather than object and relationship

spinedb_api/__init__.py

@@ -10,7 +10,6 @@
 ######################################################################################################################
 
 from .db_mapping import DatabaseMapping
-from .diff_db_mapping import DiffDatabaseMapping
 from .exception import (
     SpineDBAPIError,
     SpineIntegrityError,
@@ -33,21 +32,13 @@
     forward_sweep,
     Asterisk,
 )
-from .check_functions import (
-    check_alternative,
-    check_scenario,
-    check_scenario_alternative,
-    check_object_class,
-    check_object,
-    check_wide_relationship_class,
-    check_wide_relationship,
-    check_parameter_definition,
-    check_parameter_value,
-    check_parameter_value_list,
-)
 from .import_functions import (
     import_alternatives,
     import_data,
+    import_entity_classes,
+    import_entities,
+    import_parameter_definitions,
+    import_parameter_values,
     import_object_classes,
     import_objects,
     import_object_parameters,
@@ -59,10 +50,6 @@
     import_relationships,
     import_scenarios,
     import_scenario_alternatives,
-    import_tools,
-    import_features,
-    import_tool_features,
-    import_tool_feature_methods,
     import_metadata,
     import_object_metadata,
     import_relationship_metadata,
@@ -84,10 +71,6 @@
     export_relationships,
     export_scenario_alternatives,
     export_scenarios,
-    export_tools,
-    export_features,
-    export_tool_features,
-    export_tool_feature_methods,
 )
 from .import_mapping.import_mapping_compat import import_mapping_from_dict
 from .import_mapping.generator import get_mapped_data
@@ -114,7 +97,6 @@
 )
 from .filters.alternative_filter import apply_alternative_filter_to_parameter_value_sq
 from .filters.scenario_filter import apply_scenario_filter_to_subqueries
-from .filters.tool_filter import apply_tool_filter_to_entity_sq
 from .filters.execution_filter import apply_execution_filter
 from .filters.renamer import apply_renaming_to_parameter_definition_sq, apply_renaming_to_entity_class_sq
 from .filters.tools import (

spinedb_api/alembic/versions/6b7c994c1c61_drop_object_and_relationship_tables.py

@@ -0,0 +1,136 @@
+"""drop_object_and_relationship_tables
+
+Revision ID: 6b7c994c1c61
+Revises: 989fccf80441
+Create Date: 2023-02-09 06:48:46.585108
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from spinedb_api.helpers import naming_convention
+
+# revision identifiers, used by Alembic.
+revision = '6b7c994c1c61'
+down_revision = '989fccf80441'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'entity_class_dimension',
+        sa.Column('entity_class_id', sa.Integer(), nullable=False),
+        sa.Column('dimension_id', sa.Integer(), nullable=False),
+        sa.Column('position', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['dimension_id'],
+            ['entity_class.id'],
+            name=op.f('fk_entity_class_dimension_dimension_id_entity_class'),
+            onupdate='CASCADE',
+            ondelete='CASCADE',
+        ),
+        sa.ForeignKeyConstraint(
+            ['entity_class_id'],
+            ['entity_class.id'],
+            name=op.f('fk_entity_class_dimension_entity_class_id_entity_class'),
+            onupdate='CASCADE',
+            ondelete='CASCADE',
+        ),
+        sa.PrimaryKeyConstraint('entity_class_id', 'dimension_id', 'position', name=op.f('pk_entity_class_dimension')),
+        sa.UniqueConstraint('entity_class_id', 'dimension_id', 'position', name='uq_entity_class_dimension'),
+    )
+    op.create_table(
+        'entity_element',
+        sa.Column('entity_id', sa.Integer(), nullable=False),
+        sa.Column('entity_class_id', sa.Integer(), nullable=False),
+        sa.Column('element_id', sa.Integer(), nullable=False),
+        sa.Column('dimension_id', sa.Integer(), nullable=False),
+        sa.Column('position', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['element_id', 'dimension_id'],
+            ['entity.id', 'entity.class_id'],
+            name=op.f('fk_entity_element_element_id_entity'),
+            onupdate='CASCADE',
+            ondelete='CASCADE',
+        ),
+        sa.ForeignKeyConstraint(
+            ['entity_class_id', 'dimension_id', 'position'],
+            [
+                'entity_class_dimension.entity_class_id',
+                'entity_class_dimension.dimension_id',
+                'entity_class_dimension.position',
+            ],
+            name=op.f('fk_entity_element_entity_class_id_entity_class_dimension'),
+            onupdate='CASCADE',
+            ondelete='CASCADE',
+        ),
+        sa.ForeignKeyConstraint(
+            ['entity_id', 'entity_class_id'],
+            ['entity.id', 'entity.class_id'],
+            name=op.f('fk_entity_element_entity_id_entity'),
+            onupdate='CASCADE',
+            ondelete='CASCADE',
+        ),
+        sa.PrimaryKeyConstraint('entity_id', 'position', name=op.f('pk_entity_element')),
+    )
+    _persist_data()
+    # NOTE: some constraints are only created by the create_new_spine_database() function,
+    # not by the corresponding migration script. Thus we need to check before removing those constraints.
+    # We should avoid this in the future.
+    entity_class_constraints, entity_constraints = _get_constraints()
+    with op.batch_alter_table("entity", naming_convention=naming_convention) as batch_op:
+        for cname in ('uq_entity_idclass_id', 'uq_entity_idtype_idclass_id'):
+            if cname in entity_constraints:
+                batch_op.drop_constraint(cname, type_='unique')
+        batch_op.drop_constraint('fk_entity_type_id_entity_type', type_='foreignkey')
+        batch_op.drop_column('type_id')
+    with op.batch_alter_table("entity_class", naming_convention=naming_convention) as batch_op:
+        for cname in ('uq_entity_class_idtype_id', 'uq_entity_class_type_idname'):
+            if cname in entity_class_constraints:
+                batch_op.drop_constraint(cname, type_='unique')
+        batch_op.drop_constraint('fk_entity_class_type_id_entity_class_type', type_='foreignkey')
+        batch_op.drop_constraint('fk_entity_class_commit_id_commit', type_='foreignkey')
+        batch_op.drop_column('commit_id')
+        batch_op.drop_column('type_id')
+    op.drop_table('object_class')
+    op.drop_table('entity_class_type')
+    # op.drop_table('next_id')
+    op.drop_table('object')
+    op.drop_table('relationship_entity_class')
+    op.drop_table('relationship')
+    op.drop_table('entity_type')
+    op.drop_table('relationship_class')
+    op.drop_table('relationship_entity')
+
+
+def _get_constraints():
+    conn = op.get_bind()
+    meta = sa.MetaData(conn)
+    meta.reflect()
+    return [[c.name for c in meta.tables[tname].constraints] for tname in ["entity_class", "entity"]]
+
+
+def _persist_data():
+    conn = op.get_bind()
+    meta = sa.MetaData(conn)
+    meta.reflect()
+    ecd_items = [
+        {"entity_class_id": x["entity_class_id"], "dimension_id": x["member_class_id"], "position": x["dimension"]}
+        for x in conn.execute("SELECT * FROM relationship_entity_class")
+    ]
+    ee_items = [
+        {
+            "entity_id": x["entity_id"],
+            "entity_class_id": x["entity_class_id"],
+            "element_id": x["member_id"],
+            "dimension_id": x["member_class_id"],
+            "position": x["dimension"],
+        }
+        for x in conn.execute("SELECT * FROM relationship_entity")
+    ]
+    op.bulk_insert(meta.tables["entity_class_dimension"], ecd_items)
+    op.bulk_insert(meta.tables["entity_element"], ee_items)
+
+
+def downgrade():
+    pass

spinedb_api/alembic/versions/989fccf80441_replace_values_with_reference_to_list_.py

@@ -9,11 +9,7 @@
 from sqlalchemy import MetaData
 from sqlalchemy.sql.expression import bindparam
 from sqlalchemy.orm import sessionmaker
-from spinedb_api.check_functions import (
-    replace_default_values_with_list_references,
-    replace_parameter_values_with_list_references,
-)
-from spinedb_api.parameter_value import from_database
+from spinedb_api.parameter_value import dump_db_value, from_database, ParameterValueFormatError
 from spinedb_api.helpers import group_concat
 from spinedb_api.exception import SpineIntegrityError
 
@@ -71,3 +67,51 @@ def upgrade():
 
 def downgrade():
     pass
+
+
+def replace_default_values_with_list_references(item, parameter_value_lists, list_values):
+    parameter_value_list_id = item.get("parameter_value_list_id")
+    return _replace_values_with_list_references(
+        "parameter_definition", item, parameter_value_list_id, parameter_value_lists, list_values
+    )
+
+
+def replace_parameter_values_with_list_references(item, parameter_definitions, parameter_value_lists, list_values):
+    parameter_definition_id = item["parameter_definition_id"]
+    parameter_definition = parameter_definitions[parameter_definition_id]
+    parameter_value_list_id = parameter_definition["parameter_value_list_id"]
+    return _replace_values_with_list_references(
+        "parameter_value", item, parameter_value_list_id, parameter_value_lists, list_values
+    )
+
+
+def _replace_values_with_list_references(item_type, item, parameter_value_list_id, parameter_value_lists, list_values):
+    if parameter_value_list_id is None:
+        return False
+    if parameter_value_list_id not in parameter_value_lists:
+        raise SpineIntegrityError("Parameter value list not found.")
+    value_id_list = parameter_value_lists[parameter_value_list_id]
+    if value_id_list is None:
+        raise SpineIntegrityError("Parameter value list is empty!")
+    value_key, type_key = {
+        "parameter_value": ("value", "type"),
+        "parameter_definition": ("default_value", "default_type"),
+    }[item_type]
+    value = dict.get(item, value_key)
+    value_type = dict.get(item, type_key)
+    try:
+        parsed_value = from_database(value, value_type)
+    except ParameterValueFormatError as err:
+        raise SpineIntegrityError(f"Invalid {value_key} '{value}': {err}") from None
+    if parsed_value is None:
+        return False
+    list_value_id = next((id_ for id_ in value_id_list if list_values.get(id_) == parsed_value), None)
+    if list_value_id is None:
+        valid_values = ", ".join(f"{dump_db_value(list_values.get(id_))[0].decode('utf8')!r}" for id_ in value_id_list)
+        raise SpineIntegrityError(
+            f"Invalid {value_key} '{parsed_value}' - it should be one from the parameter value list: {valid_values}."
+        )
+    item[value_key] = str(list_value_id).encode("UTF8")
+    item[type_key] = "list_value_ref"
+    item["list_value_id"] = list_value_id
+    return True