Metadata form 7: Access control and deletion behaviour

backend/dataall/core/environment/services/environment_service.py

@@ -46,8 +46,6 @@
 from dataall.core.permissions.services.tenant_permissions import MANAGE_ENVIRONMENTS
 from dataall.core.stacks.db.stack_repositories import StackRepository
 from dataall.core.vpc.db.vpc_repositories import VpcRepository
-from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes, MetadataFormVisibility
-from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 
 log = logging.getLogger(__name__)
 
@@ -527,6 +525,7 @@ def list_group_permissions_internal(session, uri, group_uri):
         environment = EnvironmentService.get_environment_by_uri(session, uri)
 
         return ResourcePolicyService.get_resource_policy_permissions(
+            session=session,
             group_uri=group_uri,
             resource_uri=environment.environmentUri,
         )
@@ -886,12 +885,6 @@ def delete_environment(uri):
                 KeyValueTagRepository.delete_key_value_tags(session, environment.environmentUri, 'environment')
                 EnvironmentResourceManager.delete_env(session, environment)
                 EnvironmentParameterRepository(session).delete_params(environment.environmentUri)
-                MetadataFormRepository.delete_attached_entity_metadata_forms(
-                    session, environment.environmentUri, MetadataFormEntityTypes.Environments.value
-                )
-                MetadataFormRepository.delete_all_home_metadata_forms(
-                    session, environment.environmentUri, MetadataFormVisibility.Environment.value
-                )
 
                 for group in env_groups:
                     session.delete(group)

backend/dataall/core/organizations/services/organization_service.py

@@ -21,8 +21,6 @@
     ORGANIZATION_INVITED_READONLY,
     ORGANIZATION_INVITED_DESCRIPTIONS,
 )
-from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes, MetadataFormVisibility
-from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 
 
 class OrganizationService:
@@ -177,12 +175,6 @@ def archive_organization(uri):
                 resource_uri=org.organizationUri,
                 resource_type=models.Organization.__name__,
             )
-            MetadataFormRepository.delete_attached_entity_metadata_forms(
-                session, org.organizationUri, MetadataFormEntityTypes.Organizations.value
-            )
-            MetadataFormRepository.delete_all_home_metadata_forms(
-                session, org.organizationUri, MetadataFormVisibility.Organization.value
-            )
 
             return True
 
@@ -317,7 +309,11 @@ def resolve_organization_by_env(uri):
     @staticmethod
     @ResourcePolicyService.has_resource_permission(GET_ORGANIZATION)
     def list_group_organization_permissions(uri, groupUri):
-        return ResourcePolicyService.get_resource_policy_permissions(group_uri=groupUri, resource_uri=uri)
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
+            return ResourcePolicyService.get_resource_policy_permissions(
+                session=session, group_uri=groupUri, resource_uri=uri
+            )
 
     @staticmethod
     def list_invited_organization_permissions_with_descriptions():

backend/dataall/core/permissions/services/resource_policy_service.py

@@ -212,21 +212,21 @@ def associate_permission_to_resource_policy(session, policy, permission):
         session.commit()
 
     @staticmethod
-    def get_resource_policy_permissions(group_uri, resource_uri) -> List[ResourcePolicyPermission]:
+    def get_resource_policy_permissions(session, group_uri, resource_uri) -> List[ResourcePolicyPermission]:
         if not group_uri:
             raise exceptions.RequiredParameter(param_name='group_uri')
         if not resource_uri:
             raise exceptions.RequiredParameter(param_name='resource_uri')
-        with get_context().db_engine.scoped_session() as session:
-            policy = ResourcePolicyRepository.find_resource_policy(
-                session=session,
-                group_uri=group_uri,
-                resource_uri=resource_uri,
-            )
-            permissions = []
-            for p in policy.permissions:
-                permissions.append(p.permission)
-            return permissions
+
+        policy = ResourcePolicyRepository.find_resource_policy(
+            session=session,
+            group_uri=group_uri,
+            resource_uri=resource_uri,
+        )
+        permissions = []
+        for p in policy.permissions:
+            permissions.append(p.permission)
+        return permissions
 
     @staticmethod
     def has_resource_permission(

backend/dataall/modules/metadata_forms/db/metadata_form_repository.py

@@ -249,25 +249,17 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris,
         return query
 
     @staticmethod
-    def get_all_attached_metadata_forms_for_entity(session, entityUri, entityType):
-        return (
-            session.query(AttachedMetadataForm)
-            .filter(and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri))
-            .all()
+    def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType):
+        return session.query(AttachedMetadataForm).filter(
+            and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri)
         )
 
     @staticmethod
     def delete_attached_entity_metadata_forms(session, entityUri, entityType):
-        mfs = MetadataFormRepository.get_all_attached_metadata_forms_for_entity(session, entityUri, entityType)
-        for mf in mfs:
-            session.delete(mf)
+        MetadataFormRepository.query_all_attached_metadata_forms_for_entity(session, entityUri, entityType).delete()
 
     @staticmethod
     def delete_all_home_metadata_forms(session, homeEntityUri, visibility):
-        mfs = (
-            session.query(MetadataForm)
-            .filter(and_(MetadataForm.homeEntity == homeEntityUri, MetadataForm.visibility == visibility))
-            .all()
-        )
-        for mf in mfs:
-            session.delete(mf)
+        session.query(MetadataForm).filter(
+            and_(MetadataForm.homeEntity == homeEntityUri, MetadataForm.visibility == visibility)
+        ).delete()

backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py

@@ -1,10 +1,6 @@
 from dataall.base.context import get_context
 from dataall.base.db import exceptions, paginate
-from dataall.core.environment.db.environment_repositories import EnvironmentRepository
-from dataall.core.organizations.db.organization_repositories import OrganizationRepository
 from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
-from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService
-from dataall.modules.metadata_forms.db.enums import MetadataFormVisibility
 from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService
 from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM

backend/dataall/modules/metadata_forms/services/metadata_form_permissions.py

@@ -39,7 +39,12 @@
 DELETE_METADATA_FORM = 'DELETE_METADATA_FORM'
 EDIT_METADATA_FORM = 'EDIT_METADATA_FORM'
 
-METADATA_FORM_PERMISSIONS_ALL = [UPDATE_METADATA_FORM_FIELD, DELETE_METADATA_FORM_FIELD, DELETE_METADATA_FORM]
+METADATA_FORM_PERMISSIONS_ALL = [
+    UPDATE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM_FIELD,
+    DELETE_METADATA_FORM,
+    EDIT_METADATA_FORM,
+]
 
 METADATA_FORM_EDIT_PERMISSIONS = [
     EDIT_METADATA_FORM,

backend/dataall/modules/metadata_forms/services/metadata_form_service.py

@@ -2,8 +2,9 @@
 from dataall.base.db import exceptions, paginate
 from dataall.core.organizations.db.organization_repositories import OrganizationRepository
 from dataall.core.environment.db.environment_repositories import EnvironmentRepository
+from dataall.core.permissions.db.resource_policy.resource_policy_repositories import ResourcePolicyRepository
 from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
-from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService, TenantPolicyService
+from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService
 from dataall.modules.metadata_forms.db.enums import (
     MetadataFormVisibility,
     MetadataFormFieldType,
@@ -271,12 +272,11 @@ def get_mf_permissions(entityUri):
         result_permissions = []
         with context.db_engine.scoped_session() as session:
             for permissions in ALL_METADATA_FORMS_ENTITY_PERMISSIONS:
-                if ResourcePolicyService.check_user_resource_permission(
+                if ResourcePolicyRepository.has_user_resource_permission(
                     session=session,
-                    username=context.username,
                     groups=context.groups,
-                    resource_uri=entityUri,
                     permission_name=permissions,
+                    resource_uri=entityUri,
                 ):
                     result_permissions.append(permissions)
             return result_permissions

backend/dataall/modules/redshift_datasets/services/redshift_dataset_service.py

@@ -6,8 +6,6 @@
 from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService
 from dataall.core.permissions.services.group_policy_service import GroupPolicyService
 from dataall.core.environment.services.environment_service import EnvironmentService
-from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes
-from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 from dataall.modules.vote.db.vote_repositories import VoteRepository
 from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
 
@@ -186,9 +184,7 @@ def delete_redshift_dataset(uri):
             RedshiftDatasetService._delete_dataset_term_links(session, uri)
             VoteRepository.delete_votes(session, dataset.datasetUri, VOTE_REDSHIFT_DATASET_NAME)
             session.delete(dataset)
-            MetadataFormRepository.delete_attached_entity_metadata_forms(
-                session, dataset.datasetUri, MetadataFormEntityTypes.Datasets.value
-            )
+
             session.commit()
             return True
 

backend/dataall/modules/s3_datasets/services/dataset_service.py

@@ -11,8 +11,6 @@
 from dataall.core.stacks.services.stack_service import StackService
 from dataall.core.tasks.service_handlers import Worker
 from dataall.base.aws.sts import SessionHelper
-from dataall.modules.metadata_forms.db.enums import MetadataFormEntityTypes
-from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
 from dataall.modules.s3_datasets.aws.kms_dataset_client import KmsClient
 from dataall.base.context import get_context
 from dataall.core.permissions.services.group_policy_service import GroupPolicyService
@@ -447,9 +445,6 @@ def delete_dataset(uri: str, delete_from_aws: bool = False):
             if dataset.stewards:
                 ResourcePolicyService.delete_resource_policy(session=session, resource_uri=uri, group=dataset.stewards)
             DatasetRepository.delete_dataset(session, dataset)
-            MetadataFormRepository.delete_attached_entity_metadata_forms(
-                session, dataset.datasetUri, MetadataFormEntityTypes.Datasets.value
-            )
 
         if delete_from_aws:
             StackService.delete_stack(

backend/migrations/versions/075d344ae2cc_mf_triggers.py

@@ -0,0 +1,124 @@
+"""mf_triggers
+
+Revision ID: 075d344ae2cc
+Revises: 427db8f31999
+Create Date: 2024-09-13 13:12:16.951311
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '075d344ae2cc'
+down_revision = '427db8f31999'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("""
+        CREATE OR REPLACE FUNCTION org_delete_trigger_function()
+        RETURNS TRIGGER AS $$
+        BEGIN
+            -- Delete from attached_metadata_form
+            DELETE FROM attached_metadata_form 
+            WHERE "entityUri" = OLD."organizationUri" 
+              AND "entityType" = 'Organization';
+
+            -- Delete from metadata_form
+            DELETE FROM metadata_form 
+            WHERE "homeEntity" = OLD."organizationUri" 
+              AND visibility = 'Organization-Wide';
+
+            RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+
+        -- Create the trigger for organization table
+        CREATE TRIGGER org_delete_trigger
+        BEFORE DELETE ON organization
+        FOR EACH ROW
+        EXECUTE FUNCTION org_delete_trigger_function();
+    """)
+
+    op.execute("""
+        CREATE OR REPLACE FUNCTION env_delete_trigger_function()
+        RETURNS TRIGGER AS $$
+        BEGIN
+            -- Delete from attached_metadata_form
+            DELETE FROM attached_metadata_form
+            WHERE "entityUri" = OLD."environmentUri"
+              AND "entityType" = 'Environment';
+
+            -- Delete from metadata_form
+            DELETE FROM metadata_form
+            WHERE "homeEntity" = OLD."environmentUri"
+              AND visibility = 'Environment-Wide';
+
+            RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+
+        -- Create the trigger for environment table
+        CREATE TRIGGER env_delete_trigger
+        BEFORE DELETE ON environment
+        FOR EACH ROW
+        EXECUTE FUNCTION env_delete_trigger_function();
+    """)
+
+    op.execute("""
+        CREATE OR REPLACE FUNCTION dataset_delete_trigger_function()
+        RETURNS TRIGGER AS $$
+        BEGIN
+            -- Delete from attached_metadata_form
+            DELETE FROM attached_metadata_form
+            WHERE "entityUri" = OLD."datasetUri"
+              AND "entityType" = 'Dataset';
+
+            RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+
+        -- Create the trigger for dataset table
+        CREATE TRIGGER dataset_delete_trigger
+        BEFORE DELETE ON dataset
+        FOR EACH ROW
+        EXECUTE FUNCTION dataset_delete_trigger_function();
+    """)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute(
+        """
+        -- Drop the org_delete_trigger
+        DROP TRIGGER IF EXISTS org_delete_trigger ON organization;
+
+        -- Drop the org_delete_trigger_function
+        DROP FUNCTION IF EXISTS org_delete_trigger_function;
+        """
+    )
+
+    op.execute(
+        """
+        -- Drop the env_delete_trigger
+        DROP TRIGGER IF EXISTS env_delete_trigger ON environment;
+
+        -- Drop the env_delete_trigger_function
+        DROP FUNCTION IF EXISTS env_delete_trigger_function;
+        """
+    )
+
+    op.execute(
+        """
+        -- Drop the dataset_delete_trigger
+        DROP TRIGGER IF EXISTS dataset_delete_trigger ON dataset;
+
+        -- Drop the dataset_delete_trigger_function
+        DROP FUNCTION IF EXISTS dataset_delete_trigger_function;
+        """
+    )
+    # ### end Alembic commands ###

backend/migrations/versions/a991ac7a85a2_drop_remove_group_permissions.py

@@ -45,7 +45,7 @@ def upgrade():
             .all()
         )
         for group in suspicious_permissions_principals:
-            permissions = ResourcePolicyService.get_resource_policy_permissions(group, env.environmentUri)
+            permissions = ResourcePolicyService.get_resource_policy_permissions(session, group, env.environmentUri)
             permissions = [permission.name for permission in permissions if permission.name != REMOVE_ENVIRONMENT_GROUP]
             ResourcePolicyService.update_resource_policy(
                 session,