diff --git a/addons/policies/bootstrap_relationship_policies.json b/addons/policies/bootstrap_relationship_policies.json index 2c123bec6b..cdb19ecd7d 100644 --- a/addons/policies/bootstrap_relationship_policies.json +++ b/addons/policies/bootstrap_relationship_policies.json @@ -836,6 +836,45 @@ "remove-relationship" ] } + }, + + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_ASSETS_USER_DEFINED_RELATIONSHIP", + "qualifiedName": "LINK_ASSETS_USER_DEFINED_RELATIONSHIP", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "relationship-type:UserDefRelationship", + "end-one-entity-type:Referenceable", + "end-two-entity-type:Referenceable", + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } } ] } \ No newline at end of file diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 640be44ba9..2fcdd21ddd 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -156,6 +156,10 @@ public final class Constants { public static final String INPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.inputPortDataProducts"; public static final String OUTPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.outputPortDataProducts"; + public static final String UD_RELATIONSHIP_EDGE_LABEL = "__Referenceable.userDefRelationshipTo"; + public static final String UD_RELATIONSHIP_END_NAME_FROM = "userDefRelationshipFrom"; + public static final String UD_RELATIONSHIP_END_NAME_TO = "userDefRelationshipTo"; + /** * SQL property keys. */ @@ -260,6 +264,7 @@ public final class Constants { public static final String INDEX_PREFIX = "janusgraph_"; public static final String VERTEX_INDEX_NAME = INDEX_PREFIX + VERTEX_INDEX; + public static final String EDGE_INDEX_NAME = INDEX_PREFIX + EDGE_INDEX; public static final String NAME = "name"; public static final String QUALIFIED_NAME = "qualifiedName"; diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index a701510a4e..3e23999c40 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,7 +113,9 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), - ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false); + ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), + + ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100); private static final Configuration APPLICATION_PROPERTIES; diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java b/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java index 8d8cc08247..d6a0351dfa 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java @@ -28,6 +28,7 @@ public class IndexSearchParams extends SearchParams { * */ private boolean allowDeletedRelations; private boolean accessControlExclusive; + private boolean includeRelationshipAttributes; @Override public String getQuery() { @@ -79,6 +80,14 @@ public void setRelationAttributes(Set relationAttributes) { this.relationAttributes = relationAttributes; } + public boolean isIncludeRelationshipAttributes() { + return includeRelationshipAttributes; + } + + public void setIncludeRelationshipAttributes(boolean includeRelationshipAttributes) { + this.includeRelationshipAttributes = includeRelationshipAttributes; + } + @Override public String toString() { return "IndexSearchParams{" + @@ -88,6 +97,7 @@ public String toString() { ", queryString='" + queryString + '\'' + ", allowDeletedRelations=" + allowDeletedRelations + ", accessControlExclusive=" + accessControlExclusive + + ", includeRelationshipAttributes=" + includeRelationshipAttributes + ", utmTags="+ getUtmTags() + '}'; } diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java index ed1e5ded26..6a3929e4f5 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java @@ -820,6 +820,10 @@ public AtlasObjectId getNormalizedValue(Object obj) { private boolean isValidMap(Map map) { Object guid = map.get(AtlasObjectId.KEY_GUID); + if (map.containsKey(AtlasRelatedObjectId.KEY_RELATIONSHIP_ATTRIBUTES) && !map.containsKey(AtlasRelatedObjectId.KEY_RELATIONSHIP_TYPE)) { + return false; + } + if (guid != null && StringUtils.isNotEmpty(guid.toString())) { return true; } else { diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index dc567be4fe..afb0852245 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -158,6 +158,14 @@ AtlasSearchResult searchUsingBasicQuery(String query, String type, String classi */ AtlasSearchResult directIndexSearch(SearchParams searchParams) throws AtlasBaseException; + /** + * Search for direct ES query in janusgraph_edge_index + * @param searchParams Search criteria + * @return Matching entities + * @throws AtlasBaseException + */ + AtlasSearchResult directRelationshipIndexSearch(SearchParams searchParams) throws AtlasBaseException; + /** * Search for direct ES query on search logs index * @param searchParams Search criteria diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 2a3390cfc9..5c66f77e05 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -46,7 +46,6 @@ import org.apache.atlas.repository.userprofile.UserProfileService; import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.searchlog.ESSearchLogger; -import org.apache.atlas.service.FeatureFlagStore; import org.apache.atlas.stats.StatsClient; import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; @@ -976,6 +975,7 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl IndexSearchParams params = (IndexSearchParams) searchParams; RequestContext.get().setRelationAttrsForSearch(params.getRelationAttributes()); RequestContext.get().setAllowDeletedRelationsIndexsearch(params.isAllowDeletedRelations()); + RequestContext.get().setIncludeRelationshipAttributes(params.isIncludeRelationshipAttributes()); AtlasSearchResult ret = new AtlasSearchResult(); AtlasIndexQuery indexQuery; @@ -1013,6 +1013,38 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl return ret; } + @Override + public AtlasSearchResult directRelationshipIndexSearch(SearchParams searchParams) throws AtlasBaseException { + AtlasSearchResult ret = new AtlasSearchResult(); + AtlasIndexQuery indexQuery; + + ret.setSearchParameters(searchParams); + ret.setQueryType(AtlasQueryType.INDEX); + + try { + if(LOG.isDebugEnabled()){ + LOG.debug("Performing ES relationship search for the params ({})", searchParams); + } + + indexQuery = graph.elasticsearchQuery(EDGE_INDEX_NAME); + AtlasPerfMetrics.MetricRecorder elasticSearchQueryMetric = RequestContext.get().startMetricRecord("elasticSearchQueryEdge"); + DirectIndexQueryResult indexQueryResult = indexQuery.vertices(searchParams); + if (indexQueryResult == null) { + return null; + } + RequestContext.get().endMetricRecord(elasticSearchQueryMetric); + + //Note: AtlasSearchResult.entities are not supported yet + + ret.setAggregations(indexQueryResult.getAggregationMap()); + ret.setApproximateCount(indexQuery.vertexTotals()); + } catch (Exception e) { + LOG.error("Error while performing direct relationship search for the params ({}), {}", searchParams, e.getMessage()); + throw e; + } + return ret; + } + @Override public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) throws AtlasBaseException { SearchLogSearchResult ret = new SearchLogSearchResult(); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java index 52ec9840ae..4701540793 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java @@ -415,9 +415,9 @@ private void initialize(AtlasGraph graph) throws RepositoryException, IndexExcep createVertexCentricIndex(management, CLASSIFICATION_LABEL, AtlasEdgeDirection.BOTH, Arrays.asList(CLASSIFICATION_EDGE_NAME_PROPERTY_KEY, CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY)); // create edge indexes - createEdgeIndex(management, RELATIONSHIP_GUID_PROPERTY_KEY, String.class, SINGLE, true); - createEdgeIndex(management, EDGE_ID_IN_IMPORT_KEY, String.class, SINGLE, true); - createEdgeIndex(management, ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.class, SINGLE, true); + createEdgeIndex(management, RELATIONSHIP_GUID_PROPERTY_KEY, String.class, SINGLE, true, false); + createEdgeIndex(management, EDGE_ID_IN_IMPORT_KEY, String.class, SINGLE, true, false); + createEdgeIndex(management, ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.class, SINGLE, true, false); // create fulltext indexes createFullTextIndex(management, ENTITY_TEXT_PROPERTY_KEY, String.class, SINGLE); @@ -647,21 +647,22 @@ private void createIndexForAttribute(AtlasGraphManagement management, AtlasStruc createEdgeLabel(management, propertyName); } else if (isBuiltInType || isArrayOfPrimitiveType || isArrayOfEnum) { - if (isRelationshipType(atlasType)) { - createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false); + Class primitiveClassType; + boolean isStringField = false; + + if (isArrayOfEnum) { + primitiveClassType = String.class; } else { - Class primitiveClassType; - boolean isStringField = false; + primitiveClassType = isArrayOfPrimitiveType ? getPrimitiveClass(arrayElementType.getTypeName()): getPrimitiveClass(attribTypeName); + } - if (isArrayOfEnum) { - primitiveClassType = String.class; - } else { - primitiveClassType = isArrayOfPrimitiveType ? getPrimitiveClass(arrayElementType.getTypeName()): getPrimitiveClass(attribTypeName); - } + if(primitiveClassType == String.class) { + isStringField = AtlasAttributeDef.IndexType.STRING.equals(indexType); + } - if(primitiveClassType == String.class) { - isStringField = AtlasAttributeDef.IndexType.STRING.equals(indexType); - } + if (isRelationshipType(atlasType)) { + createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false, isStringField); + } else { createVertexIndex(management, propertyName, UniqueKind.NONE, primitiveClassType, cardinality, isIndexable, false, isStringField, indexTypeESConfig, indexTypeESFields); @@ -671,10 +672,10 @@ private void createIndexForAttribute(AtlasGraphManagement management, AtlasStruc } } else if (isEnumType(attributeType)) { + boolean isStringField = AtlasAttributeDef.IndexType.STRING.equals(indexType); if (isRelationshipType(atlasType)) { - createEdgeIndex(management, propertyName, String.class, cardinality, false); + createEdgeIndex(management, propertyName, String.class, cardinality, false, isStringField); } else { - boolean isStringField = AtlasAttributeDef.IndexType.STRING.equals(indexType); createVertexIndex(management, propertyName, UniqueKind.NONE, String.class, cardinality, isIndexable, false, isStringField, indexTypeESConfig, indexTypeESFields); if (uniqPropName != null) { @@ -908,21 +909,32 @@ private void createVertexCentricIndex(AtlasGraphManagement management, String ed private void createEdgeIndex(AtlasGraphManagement management, String propertyName, Class propertyClass, - AtlasCardinality cardinality, boolean createCompositeIndex) { + AtlasCardinality cardinality, boolean createCompositeIndex, boolean isStringField) { if (propertyName != null) { AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName); + boolean enforceMixedIndex = false; + //management.getGraphIndex("edge_index").getFieldKeys().stream().map(x -> x.getName()).collect(Collectors.toSet()); - if (propertyKey == null) { - propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality); + if (propertyKey != null) { + // validate property present for EDGE_INDEX or not + // there are properties like __typeName, __state which might have been added as MixedIndex for VERTEX_INDEX but not for EDGE_INDEX + // in such case, enforceMixedIndex will ensure creation of nixed index for EDDGE_INDEX for the property + enforceMixedIndex = !management.getGraphIndex(EDGE_INDEX).getFieldKeys().contains(propertyKey); + } + + if (propertyKey == null || enforceMixedIndex) { + if (propertyKey == null) { + propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality); + } if (isIndexApplicable(propertyClass, cardinality)) { if (LOG.isDebugEnabled()) { LOG.debug("Creating backing index for edge property {} of type {} ", propertyName, propertyClass.getName()); } - management.addMixedIndex(EDGE_INDEX, propertyKey, false); + management.addMixedIndex(EDGE_INDEX, propertyKey, isStringField); - LOG.info("Created backing index for edge property {} of type {} ", propertyName, propertyClass.getName()); + LOG.info("Created backing index for edge property {} of type {} on index {}", propertyName, propertyClass.getName(), EDGE_INDEX); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index a4358e2967..3ec9dddb41 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -192,7 +192,7 @@ public void deleteRelationships(Collection edges, final boolean force } continue; } - deleteEdge(edge, isInternal || forceDelete); + deleteEdge(edge, isInternal || forceDelete || isCustomRelationship(edge)); } } @@ -381,7 +381,7 @@ public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, bo // for relationship edges, inverse vertex's relationship attribute doesn't need to be updated. // only delete the reference relationship edge if (GraphHelper.isRelationshipEdge(edge)) { - deleteEdge(edge, isInternalType); + deleteEdge(edge, isInternalType || isCustomRelationship(edge)); AtlasVertex referencedVertex = entityRetriever.getReferencedEntityVertex(edge, relationshipDirection, entityVertex); if (referencedVertex != null) { @@ -398,7 +398,7 @@ public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, bo //legacy case - not a relationship edge //If deleting just the edge, reverse attribute should be updated for any references //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated - deleteEdge(edge, true, isInternalType); + deleteEdge(edge, true, isInternalType || isCustomRelationship(edge)); } } @@ -976,7 +976,8 @@ protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVe } if (edge != null) { - deleteEdge(edge, isInternalType(inVertex) && isInternalType(outVertex)); + boolean isInternal = isInternalType(inVertex) && isInternalType(outVertex); + deleteEdge(edge, isInternal || isCustomRelationship(edge)); final RequestContext requestContext = RequestContext.get(); final String outId = GraphHelper.getGuid(outVertex); @@ -1059,6 +1060,10 @@ private boolean isInternalType(final AtlasVertex instanceVertex) { return Objects.nonNull(entityType) && entityType.isInternalType(); } + private boolean isCustomRelationship(final AtlasEdge edge) { + return edge.getLabel().equals(UD_RELATIONSHIP_EDGE_LABEL); + } + private void addToPropagatedClassificationNames(AtlasVertex entityVertex, String classificationName) { if (LOG.isDebugEnabled()) { LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex)); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index afdf2825f1..78e5fb8fb2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -46,7 +46,6 @@ import org.apache.atlas.repository.graphdb.janus.JanusUtils; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; -import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasRelationshipType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; @@ -100,6 +99,8 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { private static final String END_2_DOC_ID_KEY = "end2DocId"; private static final String ES_DOC_ID_MAP_KEY = "esDocIdMap"; + private static final String UD_RELATIONSHIP_TYPE_NAME = "UserDefRelationship"; + private static Set EXCLUDE_MUTATION_REL_TYPE_NAMES = new HashSet() {{ add(REL_DOMAIN_TO_DOMAINS); add(REL_DOMAIN_TO_PRODUCTS); @@ -140,6 +141,10 @@ public AtlasRelationship create(AtlasRelationship relationship) throws AtlasBase AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1()); AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2()); + if (relationship.getTypeName().equals(UD_RELATIONSHIP_TYPE_NAME)) { + EntityGraphMapper.validateCustomRelationship(end1Vertex, end2Vertex); + } + AtlasEdge edge = createRelationship(end1Vertex, end2Vertex, relationship); AtlasRelationship ret = edge != null ? entityRetriever.mapEdgeToAtlasRelationship(edge) : null; @@ -676,6 +681,9 @@ private void validateRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex } relationshipType.getNormalizedValue(relationship); + + Map relAttrs = relationship.getAttributes(); + EntityGraphMapper.validateCustomRelationshipAttributeValueCase(relAttrs); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index f52ef2f56d..421d9e8c18 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -192,8 +192,9 @@ public class EntityGraphMapper { private boolean DEFERRED_ACTION_ENABLED = AtlasConfiguration.TASKS_USE_ENABLED.getBoolean(); private boolean DIFFERENTIAL_AUDITS = STORE_DIFFERENTIAL_AUDITS.getBoolean(); - private static final int MAX_NUMBER_OF_RETRIES = AtlasConfiguration.MAX_NUMBER_OF_RETRIES.getInt(); - private static final int CHUNK_SIZE = AtlasConfiguration.TASKS_GRAPH_COMMIT_CHUNK_SIZE.getInt(); + private static final int MAX_NUMBER_OF_RETRIES = AtlasConfiguration.MAX_NUMBER_OF_RETRIES.getInt(); + private static final int CHUNK_SIZE = AtlasConfiguration.TASKS_GRAPH_COMMIT_CHUNK_SIZE.getInt(); + private static final int UD_REL_THRESHOLD = AtlasConfiguration.ATLAS_UD_RELATIONSHIPS_MAX_COUNT.getInt(); private final GraphHelper graphHelper; private final AtlasGraph graph; @@ -1769,8 +1770,11 @@ private AtlasEdge getEdgeUsingRelationship(AttributeMutationContext ctx, EntityM AtlasEdge edge = null; + Map relationshipAttributes = getRelationshipAttributes(ctx.getValue()); + AtlasRelationship relationship = new AtlasRelationship(relationshipName, relationshipAttributes); + if (createEdge) { - edge = relationshipStore.getOrCreate(fromVertex, toVertex, new AtlasRelationship(relationshipName)); + edge = relationshipStore.getOrCreate(fromVertex, toVertex, relationship); boolean isCreated = graphHelper.getCreatedTime(edge) == RequestContext.get().getRequestTime(); if (isCreated) { @@ -1781,7 +1785,7 @@ private AtlasEdge getEdgeUsingRelationship(AttributeMutationContext ctx, EntityM } } else { - edge = relationshipStore.getRelationship(fromVertex, toVertex, new AtlasRelationship(relationshipName)); + edge = relationshipStore.getRelationship(fromVertex, toVertex, relationship); } ret = edge; } @@ -2010,6 +2014,10 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co case OUTPUT_PORT_PRODUCT_EDGE_LABEL: addInternalProductAttr(ctx, newElementsCreated, removedElements); break; + + case UD_RELATIONSHIP_EDGE_LABEL: + validateCustomRelationship(ctx, newElementsCreated, false); + break; } if (LOG.isDebugEnabled()) { @@ -2100,6 +2108,10 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext case OUTPUT_PORT_PRODUCT_EDGE_LABEL: addInternalProductAttr(ctx, newElementsCreated, null); break; + + case UD_RELATIONSHIP_EDGE_LABEL: + validateCustomRelationship(ctx, newElementsCreated, true); + break; } if (LOG.isDebugEnabled()) { @@ -2209,6 +2221,88 @@ private void addEdgesToContext(String guid, List newElementsCreated, Lis } } + public void validateCustomRelationship(AttributeMutationContext ctx, List newElements, boolean isAppend) throws AtlasBaseException { + validateCustomRelationshipCount(ctx, newElements, isAppend); + validateCustomRelationshipAttributes(ctx, newElements); + } + + public void validateCustomRelationshipCount(AttributeMutationContext ctx, List newElements, boolean isAppend) throws AtlasBaseException { + long currentSize; + boolean isEdgeDirectionIn = ctx.getAttribute().getRelationshipEdgeDirection() == AtlasRelationshipEdgeDirection.IN; + + if (isAppend) { + currentSize = ctx.getReferringVertex().getEdgesCount(isEdgeDirectionIn ? AtlasEdgeDirection.IN : AtlasEdgeDirection.OUT, + UD_RELATIONSHIP_EDGE_LABEL); + } else { + currentSize = newElements.size(); + } + + validateCustomRelationshipCount(currentSize, ctx.getReferringVertex()); + + AtlasEdgeDirection direction; + if (isEdgeDirectionIn) { + direction = AtlasEdgeDirection.OUT; + } else { + direction = AtlasEdgeDirection.IN; + } + + for (Object obj : newElements) { + AtlasEdge edge = (AtlasEdge) obj; + + AtlasVertex targetVertex; + if (isEdgeDirectionIn) { + targetVertex = edge.getOutVertex(); + LOG.info("{}: {}", direction, "outVertex"); + } else { + targetVertex = edge.getInVertex(); + LOG.info("{}: {}", direction, "inVertex"); + } + + currentSize = targetVertex.getEdgesCount(direction, UD_RELATIONSHIP_EDGE_LABEL); + validateCustomRelationshipCount(currentSize, targetVertex); + } + } + + public void validateCustomRelationshipAttributes(AttributeMutationContext ctx, List newElements) throws AtlasBaseException { + List customRelationships = (List) ctx.getValue(); + + if (CollectionUtils.isNotEmpty(customRelationships)) { + for (AtlasRelatedObjectId relatedObjectId : customRelationships) { + validateCustomRelationshipAttributeValueCase(relatedObjectId.getRelationshipAttributes().getAttributes()); + } + } + } + + public static void validateCustomRelationshipAttributeValueCase(Map attributes) throws AtlasBaseException { + if (MapUtils.isEmpty(attributes)) { + return; + } + + for (String key : attributes.keySet()) { + if (key.equals("toType") || key.equals("fromType")) { + String value = (String) attributes.get(key); + char init = value.charAt(0); + String sub = value.substring(1); + attributes.put(key, Character.toUpperCase(init) + sub.toLowerCase()); + } + } + } + + public static void validateCustomRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex) throws AtlasBaseException { + long currentSize = end1Vertex.getEdgesCount(AtlasEdgeDirection.OUT, UD_RELATIONSHIP_EDGE_LABEL) + 1; + validateCustomRelationshipCount(currentSize, end1Vertex); + + currentSize = end2Vertex.getEdgesCount(AtlasEdgeDirection.IN, UD_RELATIONSHIP_EDGE_LABEL) + 1; + validateCustomRelationshipCount(currentSize, end2Vertex); + } + + private static void validateCustomRelationshipCount(long size, AtlasVertex vertex) throws AtlasBaseException { + if (UD_REL_THRESHOLD < size) { + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, + "Custom relationships size is more than " + UD_REL_THRESHOLD + ", current is " + size + " for " + vertex.getProperty(NAME, String.class)); + } + } + private void addInternalProductAttr(AttributeMutationContext ctx, List createdElements, List deletedElements) throws AtlasBaseException { MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttrForAppend"); AtlasVertex toVertex = ctx.getReferringVertex(); @@ -2676,6 +2770,8 @@ private static Map getRelationshipAttributes(Object val) throws if (relationshipStruct instanceof Map) { return AtlasTypeUtil.toStructAttributes(((Map) relationshipStruct)); } + } else if (val instanceof AtlasObjectId) { + return ((AtlasObjectId) val).getAttributes(); } return null; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 90e041b473..60760d166a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -121,6 +121,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.*; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.isReference; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.BOTH; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; @@ -1658,6 +1659,17 @@ private AtlasObjectId mapVertexToObjectId(AtlasVertex entityVertex, String edgeL ret = toAtlasObjectId(referenceVertex); } } + + if (RequestContext.get().isIncludeRelationshipAttributes()) { + String relationshipTypeName = GraphHelper.getTypeName(edge); + boolean isRelationshipAttribute = typeRegistry.getRelationshipDefByName(relationshipTypeName) != null; + if (isRelationshipAttribute) { + AtlasRelationship relationship = mapEdgeToAtlasRelationship(edge); + Map relationshipAttributes = mapOf("typeName", relationshipTypeName); + relationshipAttributes.put("attributes", relationship.getAttributes()); + ret.getAttributes().put("relationshipAttributes", relationshipAttributes); + } + } } return ret; diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 565832b7bd..22b046bab0 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -88,6 +88,7 @@ public class RequestContext { private boolean allowDeletedRelationsIndexsearch = false; private boolean includeMeanings = true; private boolean includeClassifications = true; + private boolean includeRelationshipAttributes; private boolean includeClassificationNames = false; private String currentTypePatchAction = ""; @@ -153,6 +154,7 @@ public void clearCache() { this.onlyCAUpdateEntities.clear(); this.onlyBAUpdateEntities.clear(); this.relationAttrsForSearch.clear(); + this.includeRelationshipAttributes = false; this.queuedTasks.clear(); this.newElementsCreatedMap.clear(); this.removedElementsMap.clear(); @@ -206,6 +208,14 @@ public void setRelationAttrsForSearch(Set relationAttrsForSearch) { } } + public boolean isIncludeRelationshipAttributes() { + return includeRelationshipAttributes; + } + + public void setIncludeRelationshipAttributes(boolean includeRelationshipAttributes) { + this.includeRelationshipAttributes = includeRelationshipAttributes; + } + public Map> getRemovedElementsMap() { return removedElementsMap; } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 590e3cb0bf..5d2203b738 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -93,6 +93,7 @@ public class DiscoveryREST { private final SearchLoggingManagement loggerManagement; private static final String INDEXSEARCH_TAG_NAME = "indexsearch"; + private static final String RELATIONSHIP_INDEXSEARCH_TAG_NAME = "relationshipIndexsearch"; private static final Set TRACKING_UTM_TAGS = new HashSet<>(Arrays.asList("ui_main_list", "ui_popup_searchbar")); private static final String UTM_TAG_FROM_PRODUCT = "project_webapp"; @@ -455,6 +456,44 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } } + + /** + * Index based search for query direct on Elasticsearch Edge index + * + * @param parameters Index Search parameters @IndexSearchParams.java + * @return Atlas search result + * @throws AtlasBaseException + * @HTTP 200 On successful search + */ + @Path("/relationship/indexsearch") + @POST + @Timed + public AtlasSearchResult relationshipIndexSearch(@Context HttpServletRequest servletRequest, IndexSearchParams parameters) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.relationshipIndexSearch(" + parameters + ")"); + } + + if (StringUtils.isEmpty(parameters.getQuery())) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid search query"); + } + + if(LOG.isDebugEnabled()){ + LOG.debug("Performing relationship indexsearch for the params ({})", parameters); + } + return discoveryService.directRelationshipIndexSearch(parameters); + + } catch (AtlasBaseException abe) { + throw abe; + } catch (Exception e) { + throw new AtlasBaseException(e.getMessage(), e.getCause()); + } finally { + AtlasPerfTracer.log(perf); + } + } + /** * Index based search for query direct on ES search-logs index *