Database tables and enums for metadata forms (data-dot-all#1422)

### Feature or Bugfix
<!-- please choose -->
- Feature

### Detail
![MF
DB](https://github.com/user-attachments/assets/fb73a260-afc4-4811-850a-3a8600c4eb03)



### Relates
- data-dot-all#1065 

### Security
Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).

- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
  - Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
  - Is injection prevented by parametrizing queries?
  - Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
  - Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
  - Do you use a standard proven implementations?
  - Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
  - Have you used the least-privilege principle? How?


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Sofia Sazonova <[email protected]>

backend/dataall/modules/metadata_forms/db/enums.py

@@ -0,0 +1,47 @@
+from dataall.base.api.constants import GraphQLEnumMapper
+
+
+class MetadataFormVisibility(GraphQLEnumMapper):
+    Team = 'Team Only'
+    Environment = 'Environment-Wide'
+    Organization = 'Organization-Wide'
+    Global = 'Global'
+
+
+class MetadataFormFieldType(GraphQLEnumMapper):
+    String = 'String'
+    Integer = 'Integer'
+    Boolean = 'Boolean'
+    GlossaryTerm = 'Glossary Term'
+
+
+class MetadataFormEntityTypes(GraphQLEnumMapper):
+    Organizations = 'Organization'
+    OrganizationTeams = 'Organization Team'
+    Environments = 'Environment'
+    EnvironmentTeams = 'Environment Team'
+    Datasets = 'Dataset'
+    Worksheets = 'Worksheets'
+    Dashboards = 'Dashboard'
+    ConsumptionRoles = 'Consumption Role'
+    Notebooks = 'Notebook'
+    MLStudioEntities = 'ML Studio Entity'
+    Pipelines = 'Pipeline'
+    Tables = 'Table'
+    Folder = 'Folder'
+    Bucket = 'Bucket'
+    Share = 'Share'
+    ShareItem = 'Share Item'
+
+
+class MetadataFormEnforcementSeverity(GraphQLEnumMapper):
+    Mandatory = 'Mandatory'
+    Recommended = 'Recommended'
+
+
+class MetadataFormEnforcementScope(GraphQLEnumMapper):
+    Item = 'Item Level'
+    Dataset = 'Dataset Level'
+    Environment = 'Environmental Level'
+    Organization = 'Organizational Level'
+    Global = 'Global'

backend/dataall/modules/metadata_forms/db/metadata_form_models.py

@@ -0,0 +1,133 @@
+from sqlalchemy import (
+    Column,
+    String,
+    Integer,
+    ForeignKey,
+    Boolean,
+    ForeignKeyConstraint,
+    PrimaryKeyConstraint,
+)
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.orm import relationship, validates
+
+from dataall.base.db import Base, utils
+
+from dataall.modules.metadata_forms.db.enums import MetadataFormFieldType
+
+
+class MetadataForm(Base):
+    __tablename__ = 'metadata_form'
+    uri = Column(String, primary_key=True, default=utils.uuid('metadata_form'))
+    name = Column(String, nullable=False)
+    description = Column(String, nullable=False)
+    SamlGroupName = Column(String, nullable=False)
+    visibility = Column(String, nullable=False)  # enum MetadataFormVisibility
+    homeEntity = Column(String, nullable=True)
+
+
+class MetadataFormEnforcementRule(Base):
+    __tablename__ = 'metadata_form_enforcement_rule'
+    uri = Column(String, primary_key=True, default=utils.uuid('rule'))
+    metadataFormUri = Column(String, ForeignKey('metadata_form.uri'))
+    level = Column(String, nullable=False)  # enum MetadataFormEnforcementScope
+    entityTypes = Column(ARRAY(String), nullable=False)  # enum MetadataFormEntityTypes
+    severity = Column(String, nullable=False)  # enum MetadataFormEnforcementSeverity
+
+
+class MetadataFormField(Base):
+    __tablename__ = 'metadata_form_field'
+    metadataFormUri = Column(String, ForeignKey('metadata_form.uri'))
+    uri = Column(String, primary_key=True, default=utils.uuid('field'))
+    name = Column(String, nullable=False)
+    type = Column(String, nullable=False)  # enum MetadataFormFieldType
+    required = Column(Boolean, nullable=False)
+    glossaryNodeUri = Column(String, ForeignKey('glossary_node.nodeUri'), nullable=True)
+    possibleValues = Column(ARRAY(String), nullable=True)
+
+
+class AttachedMetadataForm(Base):
+    __tablename__ = 'attached_metadata_form'
+    metadataFormUri = Column(String, ForeignKey('metadata_form.uri'), nullable=False)
+    uri = Column(String, primary_key=True, default=utils.uuid('attached_form'))
+    entityUri = Column(String, nullable=False)
+    entityType = Column(String, nullable=False)
+
+
+class AttachedMetadataFormField(Base):
+    __tablename__ = 'attached_metadata_form_field'
+    attachedFormUri = Column(String, ForeignKey('attached_metadata_form.uri'), primary_key=True)
+    fieldUri = Column(String, ForeignKey('metadata_form_field.uri'), primary_key=True)
+    type = Column(String, nullable=False)
+    field = relationship('MetadataFormField', backref='attached_fields')
+
+    __table_args__ = (PrimaryKeyConstraint('attachedFormUri', 'fieldUri'),)
+    __mapper_args__ = {'polymorphic_identity': 'attached_metadata_form_field', 'polymorphic_on': type}
+
+    @property
+    def value(self):
+        raise NotImplementedError('Basic AttachedMetadataFormField has no implemented property value')
+
+    @validates('type')
+    def update_type(self, key, new_type):
+        if new_type != self.field.type:
+            raise ValueError("Value type doesn't match field type")
+
+
+class StringAttachedMetadataFormField(AttachedMetadataFormField):
+    __tablename__ = 'string_attached_metadata_form_field'
+    attachedFormUri = Column(String, primary_key=True)
+    fieldUri = Column(String, primary_key=True)
+    value = Column(String, nullable=False)
+    __mapper_args__ = {'polymorphic_identity': MetadataFormFieldType.String}
+
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+        ),
+    )
+
+
+class BooleanAttachedMetadataFormField(AttachedMetadataFormField):
+    __tablename__ = 'boolean_attached_metadata_form_field'
+    attachedFormUri = Column(String, primary_key=True)
+    fieldUri = Column(String, primary_key=True)
+    value = Column(Boolean, nullable=False)
+    __mapper_args__ = {'polymorphic_identity': MetadataFormFieldType.Boolean}
+
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+        ),
+    )
+
+
+class IntegerAttachedMetadataFormField(AttachedMetadataFormField):
+    __tablename__ = 'integer_attached_metadata_form_field'
+    attachedFormUri = Column(String, primary_key=True)
+    fieldUri = Column(String, primary_key=True)
+    value = Column(Integer, nullable=False)
+    __mapper_args__ = {'polymorphic_identity': MetadataFormFieldType.Integer}
+
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+        ),
+    )
+
+
+class GlossaryTermAttachedMetadataFormField(AttachedMetadataFormField):
+    __tablename__ = 'glossary_term_attached_metadata_form_field'
+    attachedFormUri = Column(String, primary_key=True)
+    fieldUri = Column(String, primary_key=True)
+    value = Column(String, nullable=False)
+    __mapper_args__ = {'polymorphic_identity': MetadataFormFieldType.GlossaryTerm}
+
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+        ),
+    )

backend/migrations/env.py

@@ -18,6 +18,7 @@
 from dataall.modules.vote.db.vote_models import Vote
 from dataall.modules.worksheets.db.worksheet_models import WorksheetQueryResult, Worksheet
 from dataall.modules.omics.db.omics_models import OmicsWorkflow, OmicsRun
+from dataall.modules.metadata_forms.db.metadata_form_models import *
 # fmt: on
 # enable ruff-format back
 

backend/migrations/versions/7c5b30fee306_metadata_forms.py

@@ -0,0 +1,135 @@
+"""metadata_forms
+
+Revision ID: 7c5b30fee306
+Revises: 797dd1012be1
+Create Date: 2024-07-19 15:03:20.671575
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '7c5b30fee306'
+down_revision = '797dd1012be1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'metadata_form',
+        sa.Column('uri', sa.String(), nullable=False),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('description', sa.String(), nullable=False),
+        sa.Column('SamlGroupName', sa.String(), nullable=False),
+        sa.Column('visibility', sa.String(), nullable=False),
+        sa.Column('homeEntity', sa.String(), nullable=True),
+        sa.PrimaryKeyConstraint('uri', name='pk_metadata_form'),
+    )
+    op.create_table(
+        'attached_metadata_form',
+        sa.Column('metadataFormUri', sa.String(), nullable=False),
+        sa.Column('uri', sa.String(), nullable=False),
+        sa.Column('entityUri', sa.String(), nullable=False),
+        sa.Column('entityType', sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(['metadataFormUri'], ['metadata_form.uri'], name='fk_attached_mf_uri'),
+        sa.PrimaryKeyConstraint('uri', name='pk_attached_metadata_form'),
+    )
+    op.create_table(
+        'metadata_form_enforcement_rule',
+        sa.Column('uri', sa.String(), nullable=False),
+        sa.Column('metadataFormUri', sa.String(), nullable=False),
+        sa.Column('level', sa.String(), nullable=False),
+        sa.Column('entityTypes', postgresql.ARRAY(sa.String()), nullable=False),
+        sa.Column('severity', sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(['metadataFormUri'], ['metadata_form.uri'], name='fk_mf_enforced_uri'),
+        sa.PrimaryKeyConstraint('uri', name='pk_metadata_form_enforcement_rule'),
+    )
+    op.create_table(
+        'metadata_form_field',
+        sa.Column('metadataFormUri', sa.String(), nullable=True),
+        sa.Column('uri', sa.String(), nullable=False),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('type', sa.String(), nullable=False),
+        sa.Column('required', sa.Boolean(), nullable=False),
+        sa.Column('glossaryNodeUri', sa.String(), nullable=True),
+        sa.Column('possibleValues', postgresql.ARRAY(sa.String()), nullable=True),
+        sa.ForeignKeyConstraint(['glossaryNodeUri'], ['glossary_node.nodeUri'], name='fk_mf_field_glossary_node'),
+        sa.ForeignKeyConstraint(['metadataFormUri'], ['metadata_form.uri'], name='fk_mf_filed_form_uri'),
+        sa.PrimaryKeyConstraint('uri', name='pk_metadata_form_field'),
+    )
+    op.create_table(
+        'attached_metadata_form_field',
+        sa.Column('attachedFormUri', sa.String(), nullable=False),
+        sa.Column('fieldUri', sa.String(), nullable=False),
+        sa.Column('type', sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(['attachedFormUri'], ['attached_metadata_form.uri'], name='fk_attached_field_mf_uri'),
+        sa.ForeignKeyConstraint(['fieldUri'], ['metadata_form_field.uri'], name='fk_attached_field_uri'),
+        sa.PrimaryKeyConstraint('attachedFormUri', 'fieldUri', name='pk_attached_metadata_form_field'),
+    )
+    op.create_table(
+        'boolean_attached_metadata_form_field',
+        sa.Column('attachedFormUri', sa.String(), nullable=False),
+        sa.Column('fieldUri', sa.String(), nullable=False),
+        sa.Column('value', sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+            name='fk_b_field',
+        ),
+        sa.PrimaryKeyConstraint('attachedFormUri', 'fieldUri', name='pk_boolean_attached_metadata_form_field'),
+    )
+    op.create_table(
+        'glossary_term_attached_metadata_form_field',
+        sa.Column('attachedFormUri', sa.String(), nullable=False),
+        sa.Column('fieldUri', sa.String(), nullable=False),
+        sa.Column('value', sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+            name='fk_gt_field',
+        ),
+        sa.PrimaryKeyConstraint('attachedFormUri', 'fieldUri', name='pk_glossary_term_attached_metadata_form_field'),
+    )
+    op.create_table(
+        'integer_attached_metadata_form_field',
+        sa.Column('attachedFormUri', sa.String(), nullable=False),
+        sa.Column('fieldUri', sa.String(), nullable=False),
+        sa.Column('value', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+            name='fk_i_field',
+        ),
+        sa.PrimaryKeyConstraint('attachedFormUri', 'fieldUri', name='pk_integer_attached_metadata_form_field'),
+    )
+    op.create_table(
+        'string_attached_metadata_form_field',
+        sa.Column('attachedFormUri', sa.String(), nullable=False),
+        sa.Column('fieldUri', sa.String(), nullable=False),
+        sa.Column('value', sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['attachedFormUri', 'fieldUri'],
+            ['attached_metadata_form_field.attachedFormUri', 'attached_metadata_form_field.fieldUri'],
+            name='fk_s_field',
+        ),
+        sa.PrimaryKeyConstraint('attachedFormUri', 'fieldUri', name='pk_string_attached_metadata_form_field'),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('string_attached_metadata_form_field')
+    op.drop_table('integer_attached_metadata_form_field')
+    op.drop_table('glossary_term_attached_metadata_form_field')
+    op.drop_table('boolean_attached_metadata_form_field')
+    op.drop_table('attached_metadata_form_field')
+    op.drop_table('metadata_form_field')
+    op.drop_table('metadata_form_enforcement_rule')
+    op.drop_table('attached_metadata_form')
+    op.drop_table('metadata_form')
+    # ### end Alembic commands ###