From 1313078fc1363e6f40717b60a904505a74518085 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 27 Aug 2024 02:21:54 +0530 Subject: [PATCH 01/20] validations for domainGUIDs --- .../apache/atlas/repository/Constants.java | 1 + .../store/graph/v2/AtlasEntityStoreV2.java | 68 +++++----- .../v2/preprocessor/AssetPreProcessor.java | 119 ++++++++++++++++++ 3 files changed, 156 insertions(+), 32 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 640be44ba9..5e483abc2f 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -269,6 +269,7 @@ public final class Constants { public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.tags.max-query-str-length"; public static final String INDEX_SEARCH_VERTEX_PREFIX_PROPERTY = "atlas.graph.index.search.vertex.prefix"; public static final String INDEX_SEARCH_VERTEX_PREFIX_DEFAULT = "$v$"; + public static final String DOMAIN_GUIDS = "domainGUIDs"; public static final String ATTR_TENANT_ID = "tenantId"; public static final String DEFAULT_TENANT_ID = "default"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index a5b168f1e7..d2c56f1b35 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -53,6 +53,7 @@ import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; import org.apache.atlas.repository.store.graph.v2.AtlasEntityComparator.AtlasEntityDiffResult; +import org.apache.atlas.repository.store.graph.v2.preprocessor.AssetPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; @@ -1552,25 +1553,25 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean private void executePreProcessor(EntityMutationContext context) throws AtlasBaseException { AtlasEntityType entityType; - PreProcessor preProcessor; + List preProcessors; List copyOfCreated = new ArrayList<>(context.getCreatedEntities()); for (AtlasEntity entity : copyOfCreated) { entityType = context.getType(entity.getGuid()); - preProcessor = getPreProcessor(entityType.getTypeName()); - - if (preProcessor != null) { - preProcessor.processAttributes(entity, context, CREATE); + preProcessors = getPreProcessor(entityType.getTypeName()); + for(PreProcessor processor : preProcessors){ + processor.processAttributes(entity, context, CREATE); } + + } List copyOfUpdated = new ArrayList<>(context.getUpdatedEntities()); for (AtlasEntity entity: copyOfUpdated) { entityType = context.getType(entity.getGuid()); - preProcessor = getPreProcessor(entityType.getTypeName()); - - if (preProcessor != null) { - preProcessor.processAttributes(entity, context, UPDATE); + preProcessors = getPreProcessor(entityType.getTypeName()); + for(PreProcessor processor : preProcessors){ + processor.processAttributes(entity, context, UPDATE); } } } @@ -1802,80 +1803,83 @@ private AtlasStruct getStarredDetailsStruct(String assetStarredBy, long assetSta return starredDetails; } - public PreProcessor getPreProcessor(String typeName) { - PreProcessor preProcessor = null; + public List getPreProcessor(String typeName) { + LinkedList preProcessors = new LinkedList<>(); switch (typeName) { case ATLAS_GLOSSARY_ENTITY_TYPE: - preProcessor = new GlossaryPreProcessor(typeRegistry, entityRetriever); + preProcessors.addFirst(new GlossaryPreProcessor(typeRegistry, entityRetriever)); break; case ATLAS_GLOSSARY_TERM_ENTITY_TYPE: - preProcessor = new TermPreProcessor(typeRegistry, entityRetriever, graph, taskManagement); + preProcessors.addFirst(new TermPreProcessor(typeRegistry, entityRetriever, graph, taskManagement)); break; case ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE: - preProcessor = new CategoryPreProcessor(typeRegistry, entityRetriever, graph, taskManagement, entityGraphMapper); + preProcessors.addFirst(new CategoryPreProcessor(typeRegistry, entityRetriever, graph, taskManagement, entityGraphMapper)); break; case DATA_DOMAIN_ENTITY_TYPE: - preProcessor = new DataDomainPreProcessor(typeRegistry, entityRetriever, graph); + preProcessors.addFirst(new DataDomainPreProcessor(typeRegistry, entityRetriever, graph)); break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, this); + preProcessors.addFirst(new DataProductPreProcessor(typeRegistry, entityRetriever, graph, this)); break; case QUERY_ENTITY_TYPE: - preProcessor = new QueryPreProcessor(typeRegistry, entityRetriever); + preProcessors.addFirst(new QueryPreProcessor(typeRegistry, entityRetriever)); break; case QUERY_FOLDER_ENTITY_TYPE: - preProcessor = new QueryFolderPreProcessor(typeRegistry, entityRetriever); + preProcessors.addFirst(new QueryFolderPreProcessor(typeRegistry, entityRetriever)); break; case QUERY_COLLECTION_ENTITY_TYPE: - preProcessor = new QueryCollectionPreProcessor(typeRegistry, discovery, entityRetriever, featureFlagStore, this); + preProcessors.addFirst(new QueryCollectionPreProcessor(typeRegistry, discovery, entityRetriever, featureFlagStore, this)); break; case PERSONA_ENTITY_TYPE: - preProcessor = new PersonaPreProcessor(graph, typeRegistry, entityRetriever, this); + preProcessors.addFirst(new PersonaPreProcessor(graph, typeRegistry, entityRetriever, this)); break; case PURPOSE_ENTITY_TYPE: - preProcessor = new PurposePreProcessor(graph, typeRegistry, entityRetriever, this); + preProcessors.addFirst(new PurposePreProcessor(graph, typeRegistry, entityRetriever, this)); break; case POLICY_ENTITY_TYPE: - preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); + preProcessors.addFirst(new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever)); break; case STAKEHOLDER_ENTITY_TYPE: - preProcessor = new StakeholderPreProcessor(graph, typeRegistry, entityRetriever, this); + preProcessors.addFirst(new StakeholderPreProcessor(graph, typeRegistry, entityRetriever, this)); break; case CONNECTION_ENTITY_TYPE: - preProcessor = new ConnectionPreProcessor(graph, discovery, entityRetriever, featureFlagStore, deleteDelegate, this); + preProcessors.addFirst(new ConnectionPreProcessor(graph, discovery, entityRetriever, featureFlagStore, deleteDelegate, this)); break; case LINK_ENTITY_TYPE: - preProcessor = new LinkPreProcessor(typeRegistry, entityRetriever); + preProcessors.addFirst(new LinkPreProcessor(typeRegistry, entityRetriever)); break; case README_ENTITY_TYPE: - preProcessor = new ReadmePreProcessor(typeRegistry, entityRetriever); + preProcessors.addFirst(new ReadmePreProcessor(typeRegistry, entityRetriever)); break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery); + preProcessors.addFirst(new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery)); break; case STAKEHOLDER_TITLE_ENTITY_TYPE: - preProcessor = new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever); + preProcessors.addFirst(new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever)); break; } - return preProcessor; + // The default global pre-processor for all AssetTypes + preProcessors.addLast(new AssetPreProcessor(typeRegistry, entityRetriever)); + + return preProcessors; } private AtlasVertex getResolvedEntityVertex(EntityGraphDiscoveryContext context, AtlasEntity entity) throws AtlasBaseException { @@ -1921,9 +1925,9 @@ private EntityMutationResponse deleteVertices(Collection deletionCa for (AtlasVertex vertex : deletionCandidates) { String typeName = getTypeName(vertex); - PreProcessor preProcessor = getPreProcessor(typeName); - if (preProcessor != null) { - preProcessor.processDelete(vertex); + List preProcessors = getPreProcessor(typeName); + for(PreProcessor processor : preProcessors){ + processor.processDelete(vertex); } if (ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE.equals(typeName)) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java new file mode 100644 index 0000000000..fa97b3f854 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -0,0 +1,119 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor ; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; + +public class AssetPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AssetPreProcessor.class); + + private EntityMutationContext context; + private AtlasTypeRegistry typeRegistry; + private EntityGraphRetriever entityRetriever; + + public AssetPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("AssetPreProcessor.processAttributes: pre processing {}, {}", + entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + this.context = context; + + AtlasEntity entity = (AtlasEntity) entityStruct; + + AtlasVertex vertex = context.getVertex(entity.getGuid()); + + switch (operation) { + case CREATE: + processCreateAsset(entity, vertex); + break; + case UPDATE: + processUpdateAsset(entity, vertex); + break; + } + } + + private void processCreateAsset(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateAsset"); + + if(entity.hasAttribute(DOMAIN_GUIDS)) { + validateDomainAssetLinks(entity); + AtlasEntityHeader sourceEntity = new AtlasEntityHeader(entity); + isAuthorized(sourceEntity); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + + private void processUpdateAsset(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateAsset"); + + if(entity.hasAttribute(DOMAIN_GUIDS)) { + validateDomainAssetLinks(entity); + AtlasEntityHeader sourceEntity = new AtlasEntityHeader(entity); + isAuthorized(sourceEntity); + } + + RequestContext.get().endMetricRecord(metricRecorder); + + } + + private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseException { + List domainGuids = ( List) entity.getAttribute(DOMAIN_GUIDS); + + if(domainGuids.size() > 1) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset can be linked to only one domain"); + } + + if(CollectionUtils.isNotEmpty(domainGuids)) { + for(String domainGuid : domainGuids) { + AtlasVertex domainVertex = context.getVertex(domainGuid); + if(domainVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, domainGuid); + } + else{ + LOG.info("Domain vertex found for guid: {}", domainGuid); + } + } + } + } + + private void isAuthorized(AtlasEntityHeader sourceEntity) throws AtlasBaseException { + + // source -> CREATE + UPDATE + READ + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, sourceEntity), + "create not allowed on source Entity: ", sourceEntity.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity), + "update not allowed on source Entity: ", sourceEntity.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, sourceEntity), + "read not allowed on source Entity: ", sourceEntity.getAttribute(NAME)); + + } + + +} From 3c9b05ef0797ef3e599b0397642d17857745da9a Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 27 Aug 2024 14:50:51 +0530 Subject: [PATCH 02/20] fetch vertex resolved --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index fa97b3f854..6ee97e7c70 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -90,7 +90,7 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept if(CollectionUtils.isNotEmpty(domainGuids)) { for(String domainGuid : domainGuids) { - AtlasVertex domainVertex = context.getVertex(domainGuid); + AtlasVertex domainVertex = entityRetriever.getEntityVertex(domainGuid); if(domainVertex == null) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, domainGuid); } From 159b375b47eec1ca12ebcfe505c36e4c035f5ab7 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 27 Aug 2024 17:14:30 +0530 Subject: [PATCH 03/20] changed permissions --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 6ee97e7c70..8320315156 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -104,9 +104,6 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept private void isAuthorized(AtlasEntityHeader sourceEntity) throws AtlasBaseException { // source -> CREATE + UPDATE + READ - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, sourceEntity), - "create not allowed on source Entity: ", sourceEntity.getAttribute(NAME)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity), "update not allowed on source Entity: ", sourceEntity.getAttribute(NAME)); From 4fc4690dce673544b98b55f380592eda232a4fe3 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 27 Aug 2024 17:44:53 +0530 Subject: [PATCH 04/20] changed permissions --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 2 -- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 865658a5a3..c71fc39132 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1574,8 +1574,6 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase for(PreProcessor processor : preProcessors){ processor.processAttributes(entity, context, CREATE); } - - } List copyOfUpdated = new ArrayList<>(context.getUpdatedEntities()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 8320315156..36cb87aa98 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -105,10 +105,10 @@ private void isAuthorized(AtlasEntityHeader sourceEntity) throws AtlasBaseExcept // source -> CREATE + UPDATE + READ AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity), - "update not allowed on source Entity: ", sourceEntity.getAttribute(NAME)); + "update on source Entity, link/unlink operation denied: ", sourceEntity.getAttribute(NAME)); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_READ, sourceEntity), - "read not allowed on source Entity: ", sourceEntity.getAttribute(NAME)); + "read on source Entity, link/unlink operation denied: ", sourceEntity.getAttribute(NAME)); } From 375d82050b569018845e05e32ea64f51d10cabc8 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 28 Aug 2024 19:32:48 +0530 Subject: [PATCH 05/20] mesh-184: refactored code --- .../store/graph/v2/AtlasEntityStoreV2.java | 38 +++++++++---------- .../v2/preprocessor/AssetPreProcessor.java | 38 +++++++++---------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index c71fc39132..5ee097eebc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1854,80 +1854,80 @@ private void createQualifiedNameHierarchyField(AtlasEntity entity, AtlasVertex v public List getPreProcessor(String typeName) { - LinkedList preProcessors = new LinkedList<>(); + List preProcessors = new ArrayList<>(); switch (typeName) { case ATLAS_GLOSSARY_ENTITY_TYPE: - preProcessors.addFirst(new GlossaryPreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new GlossaryPreProcessor(typeRegistry, entityRetriever)); break; case ATLAS_GLOSSARY_TERM_ENTITY_TYPE: - preProcessors.addFirst(new TermPreProcessor(typeRegistry, entityRetriever, graph, taskManagement)); + preProcessors.add(new TermPreProcessor(typeRegistry, entityRetriever, graph, taskManagement)); break; case ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE: - preProcessors.addFirst(new CategoryPreProcessor(typeRegistry, entityRetriever, graph, taskManagement, entityGraphMapper)); + preProcessors.add(new CategoryPreProcessor(typeRegistry, entityRetriever, graph, taskManagement, entityGraphMapper)); break; case DATA_DOMAIN_ENTITY_TYPE: - preProcessors.addFirst(new DataDomainPreProcessor(typeRegistry, entityRetriever, graph)); + preProcessors.add(new DataDomainPreProcessor(typeRegistry, entityRetriever, graph)); break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessors.addFirst(new DataProductPreProcessor(typeRegistry, entityRetriever, graph, this)); + preProcessors.add(new DataProductPreProcessor(typeRegistry, entityRetriever, graph, this)); break; case QUERY_ENTITY_TYPE: - preProcessors.addFirst(new QueryPreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new QueryPreProcessor(typeRegistry, entityRetriever)); break; case QUERY_FOLDER_ENTITY_TYPE: - preProcessors.addFirst(new QueryFolderPreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new QueryFolderPreProcessor(typeRegistry, entityRetriever)); break; case QUERY_COLLECTION_ENTITY_TYPE: - preProcessors.addFirst(new QueryCollectionPreProcessor(typeRegistry, discovery, entityRetriever, featureFlagStore, this)); + preProcessors.add(new QueryCollectionPreProcessor(typeRegistry, discovery, entityRetriever, featureFlagStore, this)); break; case PERSONA_ENTITY_TYPE: - preProcessors.addFirst(new PersonaPreProcessor(graph, typeRegistry, entityRetriever, this)); + preProcessors.add(new PersonaPreProcessor(graph, typeRegistry, entityRetriever, this)); break; case PURPOSE_ENTITY_TYPE: - preProcessors.addFirst(new PurposePreProcessor(graph, typeRegistry, entityRetriever, this)); + preProcessors.add(new PurposePreProcessor(graph, typeRegistry, entityRetriever, this)); break; case POLICY_ENTITY_TYPE: - preProcessors.addFirst(new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever)); + preProcessors.add(new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever)); break; case STAKEHOLDER_ENTITY_TYPE: - preProcessors.addFirst(new StakeholderPreProcessor(graph, typeRegistry, entityRetriever, this)); + preProcessors.add(new StakeholderPreProcessor(graph, typeRegistry, entityRetriever, this)); break; case CONNECTION_ENTITY_TYPE: - preProcessors.addFirst(new ConnectionPreProcessor(graph, discovery, entityRetriever, featureFlagStore, deleteDelegate, this)); + preProcessors.add(new ConnectionPreProcessor(graph, discovery, entityRetriever, featureFlagStore, deleteDelegate, this)); break; case LINK_ENTITY_TYPE: - preProcessors.addFirst(new LinkPreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new LinkPreProcessor(typeRegistry, entityRetriever)); break; case README_ENTITY_TYPE: - preProcessors.addFirst(new ReadmePreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new ReadmePreProcessor(typeRegistry, entityRetriever)); break; case CONTRACT_ENTITY_TYPE: - preProcessors.addFirst(new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery)); + preProcessors.add(new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery)); break; case STAKEHOLDER_TITLE_ENTITY_TYPE: - preProcessors.addFirst(new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever)); + preProcessors.add(new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever)); break; } // The default global pre-processor for all AssetTypes - preProcessors.addLast(new AssetPreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new AssetPreProcessor(typeRegistry, entityRetriever)); return preProcessors; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 36cb87aa98..344a4d42d5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -58,11 +58,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateAsset(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateAsset"); - if(entity.hasAttribute(DOMAIN_GUIDS)) { - validateDomainAssetLinks(entity); - AtlasEntityHeader sourceEntity = new AtlasEntityHeader(entity); - isAuthorized(sourceEntity); - } + processDomainLinkAttribute(entity); RequestContext.get().endMetricRecord(metricRecorder); } @@ -71,16 +67,19 @@ private void processCreateAsset(AtlasEntity entity, AtlasVertex vertex) throws A private void processUpdateAsset(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateAsset"); - if(entity.hasAttribute(DOMAIN_GUIDS)) { - validateDomainAssetLinks(entity); - AtlasEntityHeader sourceEntity = new AtlasEntityHeader(entity); - isAuthorized(sourceEntity); - } + processDomainLinkAttribute(entity); RequestContext.get().endMetricRecord(metricRecorder); } + private void processDomainLinkAttribute(AtlasEntity entity) throws AtlasBaseException { + if(entity.hasAttribute(DOMAIN_GUIDS)){ + validateDomainAssetLinks(entity); + isAuthorized(entity); + } + } + private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseException { List domainGuids = ( List) entity.getAttribute(DOMAIN_GUIDS); @@ -91,19 +90,21 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept if(CollectionUtils.isNotEmpty(domainGuids)) { for(String domainGuid : domainGuids) { AtlasVertex domainVertex = entityRetriever.getEntityVertex(domainGuid); - if(domainVertex == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, domainGuid); - } - else{ - LOG.info("Domain vertex found for guid: {}", domainGuid); - } + if(domainVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, domainGuid); + } + + if (!Objects.equals(entity.getTypeName(), DATA_DOMAIN_ENTITY_TYPE)){ + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset can be linked to only domain"); + } } } } - private void isAuthorized(AtlasEntityHeader sourceEntity) throws AtlasBaseException { + private void isAuthorized(AtlasEntity entity) throws AtlasBaseException { + AtlasEntityHeader sourceEntity = new AtlasEntityHeader(entity); - // source -> CREATE + UPDATE + READ + // source -> UPDATE + READ AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity), "update on source Entity, link/unlink operation denied: ", sourceEntity.getAttribute(NAME)); @@ -112,5 +113,4 @@ private void isAuthorized(AtlasEntityHeader sourceEntity) throws AtlasBaseExcept } - } From 3c7dde0b9a745f869c3f936f5a68ae89a4020fe1 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 28 Aug 2024 19:53:33 +0530 Subject: [PATCH 06/20] mesh-184: resolved PR comment --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 344a4d42d5..055c8ab98f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -83,11 +83,11 @@ private void processDomainLinkAttribute(AtlasEntity entity) throws AtlasBaseExce private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseException { List domainGuids = ( List) entity.getAttribute(DOMAIN_GUIDS); - if(domainGuids.size() > 1) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset can be linked to only one domain"); - } + if(CollectionUtils.isNotEmpty(domainGuids)){ + if(domainGuids.size() > 1) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset can be linked to only one domain"); + } - if(CollectionUtils.isNotEmpty(domainGuids)) { for(String domainGuid : domainGuids) { AtlasVertex domainVertex = entityRetriever.getEntityVertex(domainGuid); if(domainVertex == null) { From 5ab2f3427287791658b025d51a82fc511b50f5d7 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 29 Aug 2024 16:43:13 +0530 Subject: [PATCH 07/20] mesh-184: resolved entity error --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 055c8ab98f..9655a59c00 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -94,7 +94,9 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, domainGuid); } - if (!Objects.equals(entity.getTypeName(), DATA_DOMAIN_ENTITY_TYPE)){ + String domainEntityType = domainVertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class); + + if (!Objects.equals(domainEntityType, DATA_DOMAIN_ENTITY_TYPE)){ throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset can be linked to only domain"); } } From 50828b6bc52658bc7c9ad015d359d30648b633dd Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 9 Sep 2024 18:06:55 +0530 Subject: [PATCH 08/20] added validation on asset type --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 9655a59c00..35c612813b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -82,12 +82,17 @@ private void processDomainLinkAttribute(AtlasEntity entity) throws AtlasBaseExce private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseException { List domainGuids = ( List) entity.getAttribute(DOMAIN_GUIDS); + Set excludedTypes = new HashSet<>(Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE, ATLAS_GLOSSARY_TERM_ENTITY_TYPE, ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE, DATA_PRODUCT_ENTITY_TYPE, DATA_DOMAIN_ENTITY_TYPE)); if(CollectionUtils.isNotEmpty(domainGuids)){ if(domainGuids.size() > 1) { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset can be linked to only one domain"); } + if (excludedTypes.contains(entity.getTypeName())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset Type {} is not allowed to link with Domain", entity.getTypeName()); + } + for(String domainGuid : domainGuids) { AtlasVertex domainVertex = entityRetriever.getEntityVertex(domainGuid); if(domainVertex == null) { From 696235cbe9fcb0d06051ac51472f92b2e9239559 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 9 Sep 2024 19:52:02 +0530 Subject: [PATCH 09/20] static list for excluded assets --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 35c612813b..6204809054 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -27,6 +27,8 @@ public class AssetPreProcessor implements PreProcessor { private AtlasTypeRegistry typeRegistry; private EntityGraphRetriever entityRetriever; + private static final Set excludedTypes = new HashSet<>(Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE, ATLAS_GLOSSARY_TERM_ENTITY_TYPE, ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE, DATA_PRODUCT_ENTITY_TYPE, DATA_DOMAIN_ENTITY_TYPE)); + public AssetPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; @@ -82,7 +84,6 @@ private void processDomainLinkAttribute(AtlasEntity entity) throws AtlasBaseExce private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseException { List domainGuids = ( List) entity.getAttribute(DOMAIN_GUIDS); - Set excludedTypes = new HashSet<>(Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE, ATLAS_GLOSSARY_TERM_ENTITY_TYPE, ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE, DATA_PRODUCT_ENTITY_TYPE, DATA_DOMAIN_ENTITY_TYPE)); if(CollectionUtils.isNotEmpty(domainGuids)){ if(domainGuids.size() > 1) { From cbfa93e5f254425f6691e44952cbdd7573005ae4 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 9 Sep 2024 22:49:09 +0530 Subject: [PATCH 10/20] added validation for domain delete flow --- .../v2/preprocessor/PreProcessorUtils.java | 5 ++- .../StakeholderPreProcessor.java | 6 ++-- .../datamesh/AbstractDomainPreProcessor.java | 31 +++++++++++++++++-- .../datamesh/DataDomainPreProcessor.java | 13 ++++++++ .../AbstractGlossaryPreProcessor.java | 2 +- .../glossary/CategoryPreProcessor.java | 2 +- 6 files changed, 48 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 260f228351..a14e5747f2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -149,7 +149,7 @@ public static String updateQueryResourceAttributes(AtlasTypeRegistry typeRegistr return newCollectionQualifiedName; } - public static List indexSearchPaginated(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { + public static List indexSearchPaginated(Map dsl, Set attributes, EntityDiscoveryService discovery, int size) throws AtlasBaseException { IndexSearchParams searchParams = new IndexSearchParams(); List ret = new ArrayList<>(); @@ -163,7 +163,6 @@ public static List indexSearchPaginated(Map d dsl.put("sort", sortList); int from = 0; - int size = 100; boolean hasMore = true; do { dsl.put("from", from); @@ -196,7 +195,7 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, discovery); + List assets = indexSearchPaginated(dsl, null, discovery, 100); if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, errorMessage); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index 1cba29f935..504b90416d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -264,7 +264,7 @@ protected void verifyDuplicateStakeholderByDomainAndTitle(String domainQualified Map bool = mapOf("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, this.discovery); + List assets = indexSearchPaginated(dsl, null, this.discovery, 100); if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, @@ -282,7 +282,7 @@ protected void ensureTitleAvailableForDomain(String domainQualifiedName, String Map bool = mapOf("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, Collections.singleton(ATTR_DOMAIN_QUALIFIED_NAMES), this.discovery); + List assets = indexSearchPaginated(dsl, Collections.singleton(ATTR_DOMAIN_QUALIFIED_NAMES), this.discovery, 100); if (CollectionUtils.isNotEmpty(assets)) { AtlasEntityHeader stakeholderTitleHeader = assets.get(0); @@ -311,7 +311,7 @@ public static void verifyDuplicateStakeholderByName(String assetName, String dom Map bool = mapOf("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, discovery); + List assets = indexSearchPaginated(dsl, null, discovery, 100); if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 9f930ab831..4910164559 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -248,7 +248,7 @@ protected void exists(String assetType, String assetName, String parentDomainQua } Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, this.discovery); + List assets = indexSearchPaginated(dsl, null, this.discovery, 100); if (CollectionUtils.isNotEmpty(assets)) { for (AtlasEntityHeader asset : assets) { @@ -279,7 +279,32 @@ protected List getPolicies(Set resources) throws Atla Map dsl = mapOf("query", mapOf("bool", bool)); - return indexSearchPaginated(dsl, POLICY_ATTRIBUTES_FOR_SEARCH, discovery); + return indexSearchPaginated(dsl, POLICY_ATTRIBUTES_FOR_SEARCH, discovery, 100); + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + protected Boolean isAssetLinked(String domainGuid) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("isAssetLinked"); + boolean exists = false; + try { + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("terms", mapOf(DOMAIN_GUIDS, domainGuid))); + + Map bool = new HashMap<>(); + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List assets = indexSearchPaginated(dsl, null, this.discovery, 1); + if (CollectionUtils.isNotEmpty(assets)) { + exists = true; + } + + return exists; + } finally { RequestContext.get().endMetricRecord(metricRecorder); } @@ -302,7 +327,7 @@ protected List getStakeholderTitlesAndStakeholders(Set dsl = mapOf("query", mapOf("bool", bool)); - return indexSearchPaginated(dsl, STAKEHOLDER_ATTRIBUTES_FOR_SEARCH, discovery); + return indexSearchPaginated(dsl, STAKEHOLDER_ATTRIBUTES_FOR_SEARCH, discovery, 100); } finally { RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 929e97bdcc..78a235f3fc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -407,6 +407,19 @@ private void validateStakeholderRelationship(AtlasEntity entity) throws AtlasBas throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating/updating a domain"); } } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + String domainGuid = GraphHelper.getGuid(vertex); + + if(LOG.isDebugEnabled()) { + LOG.debug("DataDomainPreProcessor.processDelete: pre processing {}", domainGuid); + } + + if (isAssetLinked(domainGuid)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot delete domain as it has linked assets"); + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 08c604489c..7aad3985df 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -104,7 +104,7 @@ public void termExists(String termName, String glossaryQName) throws AtlasBaseEx Map dsl = mapOf("query", mapOf("bool", mapOf("must", mustClauseList))); - List terms = indexSearchPaginated(dsl, null, this.discovery); + List terms = indexSearchPaginated(dsl, null, this.discovery, 100); if (CollectionUtils.isNotEmpty(terms)) { ret = terms.stream().map(term -> (String) term.getAttribute(NAME)).anyMatch(name -> termName.equals(name)); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 88f72d2f16..6c2709f3e6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -356,7 +356,7 @@ private void categoryExists(String categoryName, String glossaryQualifiedName) t Map dsl = mapOf("query", mapOf("bool", bool)); - List categories = indexSearchPaginated(dsl, null, this.discovery); + List categories = indexSearchPaginated(dsl, null, this.discovery, 100); if (CollectionUtils.isNotEmpty(categories)) { for (AtlasEntityHeader category : categories) { From 4bb794095a79b1db019c1b33972fb47648a4d4f9 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 9 Sep 2024 23:26:17 +0530 Subject: [PATCH 11/20] Revert "added validation for domain delete flow" This reverts commit cbfa93e5f254425f6691e44952cbdd7573005ae4. --- .../v2/preprocessor/PreProcessorUtils.java | 5 +-- .../StakeholderPreProcessor.java | 6 ++-- .../datamesh/AbstractDomainPreProcessor.java | 31 ++----------------- .../datamesh/DataDomainPreProcessor.java | 13 -------- .../AbstractGlossaryPreProcessor.java | 2 +- .../glossary/CategoryPreProcessor.java | 2 +- 6 files changed, 11 insertions(+), 48 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index a14e5747f2..260f228351 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -149,7 +149,7 @@ public static String updateQueryResourceAttributes(AtlasTypeRegistry typeRegistr return newCollectionQualifiedName; } - public static List indexSearchPaginated(Map dsl, Set attributes, EntityDiscoveryService discovery, int size) throws AtlasBaseException { + public static List indexSearchPaginated(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { IndexSearchParams searchParams = new IndexSearchParams(); List ret = new ArrayList<>(); @@ -163,6 +163,7 @@ public static List indexSearchPaginated(Map d dsl.put("sort", sortList); int from = 0; + int size = 100; boolean hasMore = true; do { dsl.put("from", from); @@ -195,7 +196,7 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, discovery, 100); + List assets = indexSearchPaginated(dsl, null, discovery); if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, errorMessage); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index 504b90416d..1cba29f935 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -264,7 +264,7 @@ protected void verifyDuplicateStakeholderByDomainAndTitle(String domainQualified Map bool = mapOf("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, this.discovery, 100); + List assets = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, @@ -282,7 +282,7 @@ protected void ensureTitleAvailableForDomain(String domainQualifiedName, String Map bool = mapOf("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, Collections.singleton(ATTR_DOMAIN_QUALIFIED_NAMES), this.discovery, 100); + List assets = indexSearchPaginated(dsl, Collections.singleton(ATTR_DOMAIN_QUALIFIED_NAMES), this.discovery); if (CollectionUtils.isNotEmpty(assets)) { AtlasEntityHeader stakeholderTitleHeader = assets.get(0); @@ -311,7 +311,7 @@ public static void verifyDuplicateStakeholderByName(String assetName, String dom Map bool = mapOf("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, discovery, 100); + List assets = indexSearchPaginated(dsl, null, discovery); if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 4910164559..9f930ab831 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -248,7 +248,7 @@ protected void exists(String assetType, String assetName, String parentDomainQua } Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, this.discovery, 100); + List assets = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(assets)) { for (AtlasEntityHeader asset : assets) { @@ -279,32 +279,7 @@ protected List getPolicies(Set resources) throws Atla Map dsl = mapOf("query", mapOf("bool", bool)); - return indexSearchPaginated(dsl, POLICY_ATTRIBUTES_FOR_SEARCH, discovery, 100); - } finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - } - - protected Boolean isAssetLinked(String domainGuid) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("isAssetLinked"); - boolean exists = false; - try { - List> mustClauseList = new ArrayList<>(); - mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("terms", mapOf(DOMAIN_GUIDS, domainGuid))); - - Map bool = new HashMap<>(); - bool.put("must", mustClauseList); - - Map dsl = mapOf("query", mapOf("bool", bool)); - - List assets = indexSearchPaginated(dsl, null, this.discovery, 1); - if (CollectionUtils.isNotEmpty(assets)) { - exists = true; - } - - return exists; - + return indexSearchPaginated(dsl, POLICY_ATTRIBUTES_FOR_SEARCH, discovery); } finally { RequestContext.get().endMetricRecord(metricRecorder); } @@ -327,7 +302,7 @@ protected List getStakeholderTitlesAndStakeholders(Set dsl = mapOf("query", mapOf("bool", bool)); - return indexSearchPaginated(dsl, STAKEHOLDER_ATTRIBUTES_FOR_SEARCH, discovery, 100); + return indexSearchPaginated(dsl, STAKEHOLDER_ATTRIBUTES_FOR_SEARCH, discovery); } finally { RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 78a235f3fc..929e97bdcc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -407,19 +407,6 @@ private void validateStakeholderRelationship(AtlasEntity entity) throws AtlasBas throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating/updating a domain"); } } - - @Override - public void processDelete(AtlasVertex vertex) throws AtlasBaseException { - String domainGuid = GraphHelper.getGuid(vertex); - - if(LOG.isDebugEnabled()) { - LOG.debug("DataDomainPreProcessor.processDelete: pre processing {}", domainGuid); - } - - if (isAssetLinked(domainGuid)) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot delete domain as it has linked assets"); - } - } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 7aad3985df..08c604489c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -104,7 +104,7 @@ public void termExists(String termName, String glossaryQName) throws AtlasBaseEx Map dsl = mapOf("query", mapOf("bool", mapOf("must", mustClauseList))); - List terms = indexSearchPaginated(dsl, null, this.discovery, 100); + List terms = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(terms)) { ret = terms.stream().map(term -> (String) term.getAttribute(NAME)).anyMatch(name -> termName.equals(name)); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 6c2709f3e6..88f72d2f16 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -356,7 +356,7 @@ private void categoryExists(String categoryName, String glossaryQualifiedName) t Map dsl = mapOf("query", mapOf("bool", bool)); - List categories = indexSearchPaginated(dsl, null, this.discovery, 100); + List categories = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(categories)) { for (AtlasEntityHeader category : categories) { From 78ed97a8af0cb4a4939e856d69e72a70368348fb Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 10 Sep 2024 14:07:08 +0530 Subject: [PATCH 12/20] mesh-184: delete flow checks for domain --- .../v2/preprocessor/AssetPreProcessor.java | 2 +- .../datamesh/AbstractDomainPreProcessor.java | 54 +++++++++++++++++++ .../datamesh/DataDomainPreProcessor.java | 13 +++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 6204809054..4ebe51d820 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -91,7 +91,7 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept } if (excludedTypes.contains(entity.getTypeName())) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Asset Type {} is not allowed to link with Domain", entity.getTypeName()); + throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "This AssetType is not allowed to link with Domain", entity.getTypeName()); } for(String domainGuid : domainGuids) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 9f930ab831..33307a34af 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -25,6 +25,7 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -69,6 +70,7 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { private static final Set POLICY_ATTRIBUTES_FOR_SEARCH = new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES)); private static final Set STAKEHOLDER_ATTRIBUTES_FOR_SEARCH = new HashSet<>(Arrays.asList(ATTR_DOMAIN_QUALIFIED_NAMES, ATTR_DOMAIN_QUALIFIED_NAME)); + private static final Set DOMAIN_GUID_ATTR = new HashSet<>(Arrays.asList(DOMAIN_GUIDS)); static final Set PARENT_ATTRIBUTES = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR)); @@ -285,6 +287,58 @@ protected List getPolicies(Set resources) throws Atla } } + protected Boolean isAssetLinked(String domainGuid) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("isAssetLinked"); + boolean exists = false; + try { + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("terms", mapOf(DOMAIN_GUIDS, domainGuid))); + + Map bool = new HashMap<>(); + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List assets = fetchLinkedAssets(dsl, DOMAIN_GUID_ATTR, this.discovery); + if (CollectionUtils.isNotEmpty(assets)) { + exists = true; + } + + return exists; + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + protected static List fetchLinkedAssets(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { + IndexSearchParams searchParams = new IndexSearchParams(); + List ret = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(attributes)) { + searchParams.setAttributes(attributes); + } + + List sortList = new ArrayList<>(0); + sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); + sortList.add(mapOf("__guid", mapOf("order", "asc"))); + dsl.put("sort", sortList); + + int from = 0; + int size = 1; + dsl.put("from", from); + dsl.put("size", size); + searchParams.setDsl(dsl); + + List headers = discovery.directIndexSearch(searchParams).getEntities(); + + if (CollectionUtils.isNotEmpty(headers)) { + ret.addAll(headers); + } + return ret; + } + protected List getStakeholderTitlesAndStakeholders(Set qualifiedNames) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getStakeholderTitlesAndStakeholders"); try { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 929e97bdcc..0efd76deb3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -407,6 +407,19 @@ private void validateStakeholderRelationship(AtlasEntity entity) throws AtlasBas throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating/updating a domain"); } } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + String domainGuid = GraphHelper.getGuid(vertex); + + if(LOG.isDebugEnabled()) { + LOG.debug("DataDomainPreProcessor.processDelete: pre processing {}", domainGuid); + } + + if (isAssetLinked(domainGuid)) { + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Domain cannot be deleted because some assets are linked to this domain"); + } + } } From 2f3b4323cb3804ba0585378f0b37d1f491594f1f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 10 Sep 2024 16:22:58 +0530 Subject: [PATCH 13/20] mesh-184: indexsearch error resolved --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 33307a34af..4878c59374 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -293,7 +293,7 @@ protected Boolean isAssetLinked(String domainGuid) throws AtlasBaseException { try { List> mustClauseList = new ArrayList<>(); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("terms", mapOf(DOMAIN_GUIDS, domainGuid))); + mustClauseList.add(mapOf("term", mapOf(DOMAIN_GUIDS, domainGuid))); Map bool = new HashMap<>(); bool.put("must", mustClauseList); From 9fcd12f3715510270035a29a6fb1016ad9d93d69 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 10 Sep 2024 23:07:48 +0530 Subject: [PATCH 14/20] mesh-184: resolved PR comments --- .../datamesh/AbstractDomainPreProcessor.java | 30 +++++++------------ .../datamesh/DataDomainPreProcessor.java | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 4878c59374..fe18270a7e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -287,12 +287,11 @@ protected List getPolicies(Set resources) throws Atla } } - protected Boolean isAssetLinked(String domainGuid) throws AtlasBaseException { + protected Boolean hasLinkedAssets(String domainGuid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("isAssetLinked"); boolean exists = false; try { List> mustClauseList = new ArrayList<>(); - mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf(DOMAIN_GUIDS, domainGuid))); Map bool = new HashMap<>(); @@ -300,8 +299,8 @@ protected Boolean isAssetLinked(String domainGuid) throws AtlasBaseException { Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = fetchLinkedAssets(dsl, DOMAIN_GUID_ATTR, this.discovery); - if (CollectionUtils.isNotEmpty(assets)) { + boolean hasLinkedAsset = fetchLinkedAssets(dsl, DOMAIN_GUID_ATTR, this.discovery); + if (hasLinkedAsset) { exists = true; } @@ -312,31 +311,22 @@ protected Boolean isAssetLinked(String domainGuid) throws AtlasBaseException { } } - protected static List fetchLinkedAssets(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { + protected static Boolean fetchLinkedAssets(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { IndexSearchParams searchParams = new IndexSearchParams(); List ret = new ArrayList<>(); + boolean exists = false; - if (CollectionUtils.isNotEmpty(attributes)) { - searchParams.setAttributes(attributes); - } - - List sortList = new ArrayList<>(0); - sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); - sortList.add(mapOf("__guid", mapOf("order", "asc"))); - dsl.put("sort", sortList); - - int from = 0; - int size = 1; - dsl.put("from", from); - dsl.put("size", size); + searchParams.setAttributes(attributes); + dsl.put("from", 0); + dsl.put("size", 1); searchParams.setDsl(dsl); List headers = discovery.directIndexSearch(searchParams).getEntities(); if (CollectionUtils.isNotEmpty(headers)) { - ret.addAll(headers); + exists = true; } - return ret; + return exists; } protected List getStakeholderTitlesAndStakeholders(Set qualifiedNames) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 0efd76deb3..b4668b0868 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -416,7 +416,7 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { LOG.debug("DataDomainPreProcessor.processDelete: pre processing {}", domainGuid); } - if (isAssetLinked(domainGuid)) { + if (hasLinkedAssets(domainGuid)) { throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Domain cannot be deleted because some assets are linked to this domain"); } } From 50839bd207bcccc0f9b882ef45f8a2550fc4ba8d Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 10 Sep 2024 23:10:33 +0530 Subject: [PATCH 15/20] mesh-184: removed unused variable --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index fe18270a7e..3955955ca4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -313,7 +313,6 @@ protected Boolean hasLinkedAssets(String domainGuid) throws AtlasBaseException { protected static Boolean fetchLinkedAssets(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { IndexSearchParams searchParams = new IndexSearchParams(); - List ret = new ArrayList<>(); boolean exists = false; searchParams.setAttributes(attributes); From ce27ec3d44fbbf836ee0048efb273a25c21ca0a1 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 10 Sep 2024 23:57:05 +0530 Subject: [PATCH 16/20] mesh-184: optimised linked assets check code --- .../datamesh/AbstractDomainPreProcessor.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 3955955ca4..2d6628488f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -289,7 +289,6 @@ protected List getPolicies(Set resources) throws Atla protected Boolean hasLinkedAssets(String domainGuid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("isAssetLinked"); - boolean exists = false; try { List> mustClauseList = new ArrayList<>(); mustClauseList.add(mapOf("term", mapOf(DOMAIN_GUIDS, domainGuid))); @@ -299,19 +298,14 @@ protected Boolean hasLinkedAssets(String domainGuid) throws AtlasBaseException { Map dsl = mapOf("query", mapOf("bool", bool)); - boolean hasLinkedAsset = fetchLinkedAssets(dsl, DOMAIN_GUID_ATTR, this.discovery); - if (hasLinkedAsset) { - exists = true; - } - - return exists; + return hasLinkedAssets(dsl, DOMAIN_GUID_ATTR, this.discovery); } finally { RequestContext.get().endMetricRecord(metricRecorder); } } - protected static Boolean fetchLinkedAssets(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { + protected static Boolean hasLinkedAssets(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { IndexSearchParams searchParams = new IndexSearchParams(); boolean exists = false; From 5ee866296b617da6be45427d565942a35f552043 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Sep 2024 17:26:12 +0530 Subject: [PATCH 17/20] mesh-192: access via tags error --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../v2/preprocessor/AssetPreProcessor.java | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 5ee097eebc..89570db11a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1927,7 +1927,7 @@ public List getPreProcessor(String typeName) { } // The default global pre-processor for all AssetTypes - preProcessors.add(new AssetPreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new AssetPreProcessor(typeRegistry, entityRetriever, graph)); return preProcessors; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 4ebe51d820..698c08aad8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -7,6 +7,7 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; @@ -26,12 +27,15 @@ public class AssetPreProcessor implements PreProcessor { private EntityMutationContext context; private AtlasTypeRegistry typeRegistry; private EntityGraphRetriever entityRetriever; + private EntityGraphRetriever retrieverNoRelation = null; + private static final Set excludedTypes = new HashSet<>(Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE, ATLAS_GLOSSARY_TERM_ENTITY_TYPE, ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE, DATA_PRODUCT_ENTITY_TYPE, DATA_DOMAIN_ENTITY_TYPE)); - public AssetPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { + public AssetPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; + this.retrieverNoRelation = new EntityGraphRetriever(graph, typeRegistry, true); } @Override @@ -60,7 +64,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateAsset(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateAsset"); - processDomainLinkAttribute(entity); + processDomainLinkAttribute(entity, vertex); RequestContext.get().endMetricRecord(metricRecorder); } @@ -69,16 +73,16 @@ private void processCreateAsset(AtlasEntity entity, AtlasVertex vertex) throws A private void processUpdateAsset(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateAsset"); - processDomainLinkAttribute(entity); + processDomainLinkAttribute(entity, vertex); RequestContext.get().endMetricRecord(metricRecorder); } - private void processDomainLinkAttribute(AtlasEntity entity) throws AtlasBaseException { + private void processDomainLinkAttribute(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { if(entity.hasAttribute(DOMAIN_GUIDS)){ validateDomainAssetLinks(entity); - isAuthorized(entity); + isAuthorized(vertex); } } @@ -109,8 +113,8 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept } } - private void isAuthorized(AtlasEntity entity) throws AtlasBaseException { - AtlasEntityHeader sourceEntity = new AtlasEntityHeader(entity); + private void isAuthorized(AtlasVertex vertex) throws AtlasBaseException { + AtlasEntityHeader sourceEntity = retrieverNoRelation.toAtlasEntityHeader(vertex); // source -> UPDATE + READ AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity), From 8275720d024488b2ec43ba4564b3b68614e4c7f1 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Sep 2024 18:24:54 +0530 Subject: [PATCH 18/20] mesh-184: access via tags error debug --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index 698c08aad8..ab8330c34c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -114,7 +114,7 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept } private void isAuthorized(AtlasVertex vertex) throws AtlasBaseException { - AtlasEntityHeader sourceEntity = retrieverNoRelation.toAtlasEntityHeader(vertex); + AtlasEntityHeader sourceEntity = entityRetriever.toAtlasEntityHeader(vertex); // source -> UPDATE + READ AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity), From 79887fbb57d5473f6ef615fec6a279ab071ed475 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Sep 2024 18:39:31 +0530 Subject: [PATCH 19/20] mesh-184: access via tags error debug --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index ab8330c34c..a45871db63 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -114,7 +114,7 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept } private void isAuthorized(AtlasVertex vertex) throws AtlasBaseException { - AtlasEntityHeader sourceEntity = entityRetriever.toAtlasEntityHeader(vertex); + AtlasEntityHeader sourceEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); // source -> UPDATE + READ AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity), From b7679f57feda4a0d125b5111f19b8b182fcd5999 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Sep 2024 18:40:02 +0530 Subject: [PATCH 20/20] mesh-184: access via tags error debug --- .../store/graph/v2/preprocessor/AssetPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java index a45871db63..d32fed4733 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AssetPreProcessor.java @@ -114,7 +114,7 @@ private void validateDomainAssetLinks(AtlasEntity entity) throws AtlasBaseExcept } private void isAuthorized(AtlasVertex vertex) throws AtlasBaseException { - AtlasEntityHeader sourceEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex); + AtlasEntityHeader sourceEntity = retrieverNoRelation.toAtlasEntityHeaderWithClassifications(vertex); // source -> UPDATE + READ AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceEntity),