Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Metadata form 7: Access control and deletion behaviour #1540

Merged
merged 29 commits into from
Sep 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
719d86a
when entity deleted, all attached MFs are deleted
Sep 11, 2024
9c83eda
when env/org is deleted, all MFs with visibility inside this env/org …
Sep 11, 2024
232cf67
check resource permissions when create/attach/delete MF
Sep 11, 2024
1ec4b91
migration to backfill MF resource permissions
Sep 11, 2024
3d5242d
migrations fix
Sep 12, 2024
1cdaa06
only owners of the orgs/envs have permissions for MF
Sep 12, 2024
e267e5c
add resource policies for MFs to ENV, ORG and DATASET policy lists
Sep 12, 2024
84250f8
frontend canEdit depends on resource policies
Sep 12, 2024
84a8067
linting
Sep 12, 2024
96828ff
PR comments
Sep 13, 2024
dabc41c
instead of connecting core and modules via logic -- DB triggers
Sep 13, 2024
9645643
PR changes
Sep 13, 2024
f63d539
revert get_resource_policy_permissions change
Sep 13, 2024
52d4dde
Merge branch 'mda-main' into metadata-form-7
Sep 16, 2024
cb91ae8
share improvements and bugfixes
Sep 16, 2024
3c6926c
Merge branch 'share-ram-fix' into metadata-form-7
Sep 16, 2024
d04d005
or replace
Sep 19, 2024
ea578df
another trigger
Sep 20, 2024
82404de
backfill MF permission downgrade
Sep 20, 2024
0642d69
remove unused
Sep 20, 2024
0fbccd9
SamlAdminGroupName
Sep 20, 2024
9b1153f
check ATTACH_METADATA_FORM via decorator
Sep 23, 2024
1c6ef9c
no more arrays set to None
Sep 23, 2024
7a04e7d
ruff
Sep 23, 2024
a2f4cee
problem queries
Sep 24, 2024
5719483
fix downgrade
Sep 24, 2024
7b6023d
fix downgrade for redshift
Sep 24, 2024
0e23814
permision => resource_pol_permission.permission
Sep 26, 2024
cf5b538
Merge branch 'mda-main' into metadata-form-7
Sep 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ def get_resource_policy_permissions(session, group_uri, resource_uri) -> List[Re
raise exceptions.RequiredParameter(param_name='group_uri')
if not resource_uri:
raise exceptions.RequiredParameter(param_name='resource_uri')

policy = ResourcePolicyRepository.find_resource_policy(
session=session,
group_uri=group_uri,
Expand Down
9 changes: 9 additions & 0 deletions backend/dataall/modules/metadata_forms/api/queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
get_metadata_form,
get_attached_metadata_form,
list_attached_forms,
get_entity_metadata_form_permissions,
)

listUserMetadataForms = gql.QueryField(
Expand Down Expand Up @@ -47,3 +48,11 @@
resolver=get_attached_metadata_form,
test_scope='MetadataForm',
)

getEntityMetadataFormPermissions = gql.QueryField(
name='getEntityMetadataFormPermissions',
args=[gql.Argument('entityUri', gql.NonNullableType(gql.String))],
type=gql.ArrayType(gql.String),
resolver=get_entity_metadata_form_permissions,
test_scope='MetadataForm',
)
4 changes: 4 additions & 0 deletions backend/dataall/modules/metadata_forms/api/resolvers.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,7 @@ def has_tenant_permissions_for_metadata_forms(context: Context, source: Metadata

def resolve_metadata_form_field(context: Context, source: AttachedMetadataFormField):
return MetadataFormService.get_metadata_form_field_by_uri(uri=source.fieldUri)


def get_entity_metadata_form_permissions(context: Context, source, entityUri):
return MetadataFormService.get_mf_permissions(entityUri=entityUri)
noah-paige marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ def query_user_metadata_forms(session, is_da_admin, groups, env_uris, org_uris,
:param filter:
"""

env_uris = env_uris or []
org_uris = org_uris or []

query = session.query(MetadataForm)

if not is_da_admin:
Expand Down Expand Up @@ -141,16 +144,22 @@ def query_entity_metadata_forms(
entity_orgs_uris = entity_orgs_uris or []
entity_envs_uris = entity_envs_uris or []

orgs = list(set(user_org_uris).intersection(set(entity_orgs_uris)))
envs = list(set(user_env_uris).intersection(set(entity_envs_uris)))

query = MetadataFormRepository.query_user_metadata_forms(session, is_da_admin, groups, envs, orgs, filter)

if not orgs:
query = query.filter(MetadataForm.visibility != MetadataFormVisibility.Organization.value)
query = MetadataFormRepository.query_user_metadata_forms(
session, is_da_admin, groups, user_env_uris, user_org_uris, filter
)

if not envs:
query = query.filter(MetadataForm.visibility != MetadataFormVisibility.Environment.value)
query = query.filter(
and_(
or_(
MetadataForm.visibility != MetadataFormVisibility.Organization.value,
MetadataForm.homeEntity.in_(entity_orgs_uris),
),
or_(
MetadataForm.visibility != MetadataFormVisibility.Environment.value,
MetadataForm.homeEntity.in_(entity_envs_uris),
),
)
)

query = MetadataFormRepository.exclude_attached(session, query, filter)
return query.order_by(MetadataForm.name)
Expand Down Expand Up @@ -247,3 +256,9 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris,
if filter and filter.get('metadataFormUri'):
query = query.filter(AttachedMetadataForm.metadataFormUri == filter.get('metadataFormUri'))
return query

@staticmethod
def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType):
return session.query(AttachedMetadataForm).filter(
and_(AttachedMetadataForm.entityType == entityType, AttachedMetadataForm.entityUri == entityUri)
)
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
from dataall.base.context import get_context
from dataall.base.db import exceptions, paginate
from dataall.core.environment.db.environment_repositories import EnvironmentRepository
from dataall.core.organizations.db.organization_repositories import OrganizationRepository
from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService
from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService
from dataall.modules.metadata_forms.services.metadata_form_permissions import ATTACH_METADATA_FORM


class AttachedMetadataFormValidationService:
Expand Down Expand Up @@ -33,15 +32,22 @@ def validate_enrich_fields_params(mf_fields, data):

class AttachedMetadataFormService:
@staticmethod
def _get_entity_uri(data):
return data.get('entityUri')

@staticmethod
@ResourcePolicyService.has_resource_permission(
ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'
)
def create_attached_metadata_form(uri, data):
AttachedMetadataFormValidationService.validate_filled_form_params(uri, data)
with get_context().db_engine.scoped_session() as session:
context = get_context()
with context.db_engine.scoped_session() as session:
mf = MetadataFormRepository.get_metadata_form(session, uri)
if not mf:
raise exceptions.ObjectNotFound('MetadataForm', uri)
mf_fields = MetadataFormRepository.get_metadata_form_fields(session, uri)
AttachedMetadataFormValidationService.validate_enrich_fields_params(mf_fields, data)

amf = MetadataFormRepository.create_attached_metadata_form(session, uri, data)
for f in data.get('fields'):
MetadataFormRepository.create_attached_metadata_form_field(
Expand Down Expand Up @@ -74,7 +80,11 @@ def list_attached_forms(filter=None):
).to_dict()

@staticmethod
@ResourcePolicyService.has_resource_permission(
ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'
)
def delete_attached_metadata_form(uri):
mf = AttachedMetadataFormService.get_attached_metadata_form(uri)
with get_context().db_engine.scoped_session() as session:
context = get_context()
with context.db_engine.scoped_session() as session:
return session.delete(mf)
Original file line number Diff line number Diff line change
@@ -1,6 +1,55 @@
from dataall.core.permissions.services.environment_permissions import ENVIRONMENT_INVITED, ENVIRONMENT_ALL
from dataall.core.permissions.services.organization_permissions import (
ORGANIZATION_ALL,
ORGANIZATION_INVITED_DESCRIPTIONS,
)
from dataall.core.permissions.services.tenant_permissions import TENANT_ALL, TENANT_ALL_WITH_DESC
from dataall.core.permissions.services.resources_permissions import RESOURCES_ALL, RESOURCES_ALL_WITH_DESC
from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_WRITE, DATASET_ALL


# ------------------------TENANT-----------------------------------
MANAGE_METADATA_FORMS = 'MANAGE_METADATA_FORMS'
TENANT_ALL.append(MANAGE_METADATA_FORMS)
TENANT_ALL_WITH_DESC[MANAGE_METADATA_FORMS] = 'Manage metadata forms'

# ------------------------RESOURCE---------------------------------
# permissions to attach MF to the entity, ot make the entity the visibility base for MF
# these permissions are attached to Organizations, Environments, Datasets etc.
ATTACH_METADATA_FORM = 'ATTACH_METADATA_FORM'
CREATE_METADATA_FORM = 'CREATE_METADATA_FORM'
ALL_METADATA_FORMS_ENTITY_PERMISSIONS = [ATTACH_METADATA_FORM, CREATE_METADATA_FORM]
RESOURCES_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
RESOURCES_ALL_WITH_DESC[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope'
RESOURCES_ALL_WITH_DESC[ATTACH_METADATA_FORM] = 'Attach metadata form'

ORGANIZATION_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
ORGANIZATION_INVITED_DESCRIPTIONS[CREATE_METADATA_FORM] = 'Create metadata form within this visibility scope'
ORGANIZATION_INVITED_DESCRIPTIONS[ATTACH_METADATA_FORM] = 'Attach metadata form'

ENVIRONMENT_INVITED.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
ENVIRONMENT_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)

DATASET_WRITE.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
DATASET_ALL.extend(ALL_METADATA_FORMS_ENTITY_PERMISSIONS)
# ------------------------METADATA FORM----------------------------
# permissions to change and delete metadata forms
# these permissions are attached to MFs
UPDATE_METADATA_FORM_FIELD = 'UPDATE_METADATA_FORM_FIELD'
DELETE_METADATA_FORM_FIELD = 'DELETE_METADATA_FORM_FIELD'
DELETE_METADATA_FORM = 'DELETE_METADATA_FORM'
EDIT_METADATA_FORM = 'EDIT_METADATA_FORM'

METADATA_FORM_PERMISSIONS_ALL = [
UPDATE_METADATA_FORM_FIELD,
DELETE_METADATA_FORM_FIELD,
DELETE_METADATA_FORM,
EDIT_METADATA_FORM,
]

METADATA_FORM_EDIT_PERMISSIONS = [
EDIT_METADATA_FORM,
UPDATE_METADATA_FORM_FIELD,
DELETE_METADATA_FORM_FIELD,
]

RESOURCES_ALL.extend(METADATA_FORM_PERMISSIONS_ALL)
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,24 @@
from dataall.base.db import exceptions, paginate
from dataall.core.organizations.db.organization_repositories import OrganizationRepository
from dataall.core.environment.db.environment_repositories import EnvironmentRepository
from dataall.core.permissions.services.tenant_policy_service import TenantPolicyValidationService, TenantPolicyService
from dataall.core.permissions.db.resource_policy.resource_policy_repositories import ResourcePolicyRepository
from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService
from dataall.modules.metadata_forms.db.enums import (
MetadataFormVisibility,
MetadataFormFieldType,
)
from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
from dataall.modules.metadata_forms.services.metadata_form_access_service import MetadataFormAccessService
from dataall.modules.metadata_forms.services.metadata_form_permissions import MANAGE_METADATA_FORMS
from dataall.modules.metadata_forms.services.metadata_form_permissions import (
MANAGE_METADATA_FORMS,
DELETE_METADATA_FORM,
DELETE_METADATA_FORM_FIELD,
UPDATE_METADATA_FORM_FIELD,
CREATE_METADATA_FORM,
ALL_METADATA_FORMS_ENTITY_PERMISSIONS,
)


class MetadataFormParamValidationService:
Expand Down Expand Up @@ -91,7 +100,20 @@ class MetadataFormService:
@TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
def create_metadata_form(data):
MetadataFormParamValidationService.validate_create_form_params(data)
with get_context().db_engine.scoped_session() as session:
context = get_context()
with context.db_engine.scoped_session() as session:
if data.get('visibility') in [
MetadataFormVisibility.Organization.value,
MetadataFormVisibility.Environment.value,
]:
ResourcePolicyService.check_user_resource_permission(
session=session,
username=context.username,
groups=context.groups,
resource_uri=data.get('homeEntity'),
permission_name=CREATE_METADATA_FORM,
)

form = MetadataFormRepository.create_metadata_form(session, data)
return form

Expand All @@ -104,7 +126,7 @@ def get_metadata_form_by_uri(uri):
# toDo: deletion logic
@staticmethod
@TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
@MetadataFormAccessService.can_perform('DELETE')
@MetadataFormAccessService.can_perform(DELETE_METADATA_FORM)
def delete_metadata_form_by_uri(uri):
if mf := MetadataFormService.get_metadata_form_by_uri(uri):
with get_context().db_engine.scoped_session() as session:
Expand Down Expand Up @@ -181,15 +203,15 @@ def get_metadata_form_field_by_uri(uri):

@staticmethod
@TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
@MetadataFormAccessService.can_perform('ADD FIELD')
@MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
def create_metadata_form_field(uri, data):
MetadataFormParamValidationService.validate_create_field_params(data)
with get_context().db_engine.scoped_session() as session:
return MetadataFormRepository.create_metadata_form_field(session, uri, data)

@staticmethod
@TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
@MetadataFormAccessService.can_perform('ADD FIELDS')
@MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
def create_metadata_form_fields(uri, data_arr):
fields = []
for data in data_arr:
Expand All @@ -198,15 +220,15 @@ def create_metadata_form_fields(uri, data_arr):

@staticmethod
@TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
@MetadataFormAccessService.can_perform('DELETE FIELD')
@MetadataFormAccessService.can_perform(DELETE_METADATA_FORM_FIELD)
def delete_metadata_form_field(uri, fieldUri):
mf = MetadataFormService.get_metadata_form_field_by_uri(fieldUri)
with get_context().db_engine.scoped_session() as session:
return session.delete(mf)

@staticmethod
@TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
@MetadataFormAccessService.can_perform('UPDATE FIELDS')
@MetadataFormAccessService.can_perform('UPDATE_METADATA_FORM_FIELD')
def batch_metadata_form_field_update(uri, data):
to_delete = []
to_update = []
Expand Down Expand Up @@ -238,8 +260,23 @@ def batch_metadata_form_field_update(uri, data):

@staticmethod
@TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
@MetadataFormAccessService.can_perform('UPDATE FIELD')
@MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
def update_metadata_form_field(uri, fieldUri, data):
with get_context().db_engine.scoped_session() as session:
MetadataFormParamValidationService.validate_update_field_params(uri, data)
return MetadataFormRepository.update_metadata_form_field(session, fieldUri, data)

@staticmethod
def get_mf_permissions(entityUri):
context = get_context()
result_permissions = []
with context.db_engine.scoped_session() as session:
for permissions in ALL_METADATA_FORMS_ENTITY_PERMISSIONS:
if ResourcePolicyRepository.has_user_resource_permission(
session=session,
groups=context.groups,
permission_name=permissions,
resource_uri=entityUri,
):
result_permissions.append(permissions)
return result_permissions
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ def delete_redshift_dataset(uri):
RedshiftDatasetService._delete_dataset_term_links(session, uri)
VoteRepository.delete_votes(session, dataset.datasetUri, VOTE_REDSHIFT_DATASET_NAME)
session.delete(dataset)

session.commit()
return True

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
from dataall.modules.s3_datasets.db.dataset_bucket_repositories import DatasetBucketRepository
from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository
from dataall.modules.shares_base.services.share_object_service import ShareObjectService
from dataall.modules.vote.db.vote_repositories import VoteRepository
from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler
from dataall.modules.s3_datasets.aws.s3_dataset_client import S3DatasetClient
Expand Down
Loading
Loading