Skip to content

Commit

Permalink
A comprehensive error handling strategy of pulling entities -- part I (…
Browse files Browse the repository at this point in the history 
…#526)

* A comprehensive error handling strategy of pulling entities -- Part I

* update api param annotation

* use boolean and remove unnecessary endpoint params

* code refactor

* code refactor

* push excludeInvalid down to getEntityProjectFiles

* clean code

* resolve code reviews
  • Loading branch information
@YannanGao-gs
YannanGao-gs authored Oct 7, 2022
1 parent 0fa49a1 commit 3da555a
Show file tree
Hide file tree
Showing 20 changed files with 165 additions and 60 deletions.
7 changes: 6 additions & 1 deletion ...ver/src/main/java/org/finos/legend/sdlc/server/domain/api/entity/EntityAccessContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@ public interface EntityAccessContext
{
Entity getEntity(String path);

List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate);
default List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate)
{
return getEntities(entityPathPredicate, classifierPathPredicate, entityContentPredicate, false);
}

List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate, boolean excludeInvalid);

List<String> getEntityPaths(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate);
}
56 changes: 50 additions & 6 deletions ...nd-sdlc-server/src/main/java/org/finos/legend/sdlc/server/gitlab/api/GitLabEntityApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,12 @@ public Entity getEntity(String path)
{
try
{
return sourceDirectory.deserialize(file);
Entity localEntity = sourceDirectory.deserialize(file);
if (!Objects.equals(localEntity.getPath(), path))
{
throw new RuntimeException("Expected entity path " + path + ", found " + localEntity.getPath());
}
return localEntity;
}
catch (Exception e)
{
Expand All @@ -390,11 +395,21 @@ public Entity getEntity(String path)
}

@Override
public List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate)
public List<Entity> getEntities(Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> entityContentPredicate, boolean excludeInvalid)
{
try (Stream<EntityProjectFile> stream = getEntityProjectFiles(getFileAccessContext(getProjectFileAccessProvider()), entityPathPredicate, classifierPathPredicate, entityContentPredicate))
try (Stream<EntityProjectFile> stream = getEntityProjectFiles(getFileAccessContext(getProjectFileAccessProvider()), entityPathPredicate, classifierPathPredicate, entityContentPredicate, excludeInvalid))
{
return stream.map(EntityProjectFile::getEntity).collect(Collectors.toList());
return stream.map(excludeInvalid ? epf ->
{
try
{
return epf.getEntity();
}
catch (Exception ignore)
{
return null;
}
} : EntityProjectFile::getEntity).filter(Objects::nonNull).collect(Collectors.toList());
}
catch (Exception e)
{
Expand Down Expand Up @@ -706,6 +721,11 @@ private Stream<EntityProjectFile> getEntityProjectFiles(String projectId, String
}

private Stream<EntityProjectFile> getEntityProjectFiles(ProjectFileAccessProvider.FileAccessContext accessContext, Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> contentPredicate)
{
return getEntityProjectFiles(accessContext, entityPathPredicate, classifierPathPredicate, contentPredicate, false);
}

private Stream<EntityProjectFile> getEntityProjectFiles(ProjectFileAccessProvider.FileAccessContext accessContext, Predicate<String> entityPathPredicate, Predicate<String> classifierPathPredicate, Predicate<? super Map<String, ?>> contentPredicate, boolean excludeInvalid)
{
Stream<EntityProjectFile> stream = getEntityProjectFiles(accessContext);
if (entityPathPredicate != null)
Expand All @@ -714,11 +734,35 @@ private Stream<EntityProjectFile> getEntityProjectFiles(ProjectFileAccessProvide
}
if (classifierPathPredicate != null)
{
stream = stream.filter(epf -> classifierPathPredicate.test(epf.getEntity().getClassifierPath()));
stream = stream.filter(excludeInvalid ? epf ->
{
Entity entity;
try
{
entity = epf.getEntity();
}
catch (Exception ignore)
{
return false;
}
return classifierPathPredicate.test(entity.getClassifierPath());
} : epf -> classifierPathPredicate.test(epf.getEntity().getClassifierPath()));
}
if (contentPredicate != null)
{
stream = stream.filter(epf -> contentPredicate.test(epf.getEntity().getContent()));
stream = stream.filter(excludeInvalid ? epf ->
{
Entity entity;
try
{
entity = epf.getEntity();
}
catch (Exception ignore)
{
return false;
}
return contentPredicate.test(entity.getContent());
} : epf -> contentPredicate.test(epf.getEntity().getContent()));
}
return stream;
}
Expand Down
7 changes: 5 additions & 2 deletions ...src/main/java/org/finos/legend/sdlc/server/resources/BackupWorkspaceEntitiesResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,14 @@ public List<Entity> getAllEntities(@PathParam("projectId") String projectId,
@QueryParam("stereotype")
@ApiParam("Only include entities with one of these stereotypes. The syntax is PROFILE.NAME, where PROFILE is the full path of the Profile that owns the Stereotype.") Set<String> stereotypes,
@QueryParam("taggedValue")
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes)
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes,
@QueryParam("excludeInvalid")
@DefaultValue("false")
@ApiParam("If true, exclude invalid entities and return valid entities only. If false, the endpoint will return an error if there are any invalid entities.") boolean excludeInvalid)
{
return executeWithLogging(
"getting entities in backup user workspace " + workspaceId + " for project " + projectId,
() -> getEntities(this.entityApi.getBackupUserWorkspaceEntityAccessContext(projectId, workspaceId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes)
() -> getEntities(this.entityApi.getBackupUserWorkspaceEntityAccessContext(projectId, workspaceId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes, excludeInvalid)
);
}

Expand Down
7 changes: 5 additions & 2 deletions .../java/org/finos/legend/sdlc/server/resources/BackupWorkspaceRevisionEntitiesResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,14 @@ public List<Entity> getAllEntities(@PathParam("projectId") String projectId,
@QueryParam("stereotype")
@ApiParam("Only include entities with one of these stereotypes. The syntax is PROFILE.NAME, where PROFILE is the full path of the Profile that owns the Stereotype.") Set<String> stereotypes,
@QueryParam("taggedValue")
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes)
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes,
@QueryParam("excludeInvalid")
@DefaultValue("false")
@ApiParam("If true, exclude invalid entities and return valid entities only. If false, the endpoint will return an error if there are any invalid entities.") boolean excludeInvalid)
{
return executeWithLogging(
"getting entities in revision " + revisionId + " of backup user workspace " + workspaceId + " for project " + projectId,
() -> getEntities(this.entityApi.getBackupUserWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes)
() -> getEntities(this.entityApi.getBackupUserWorkspaceRevisionEntityAccessContext(projectId, workspaceId, revisionId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes, excludeInvalid)
);
}

Expand Down
14 changes: 10 additions & 4 deletions ...rc/main/java/org/finos/legend/sdlc/server/resources/ComparisonReviewEntitiesResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,14 @@ public List<Entity> getReviewFromEntities(@PathParam("projectId") String project
@QueryParam("stereotype")
@ApiParam("Only include entities with one of these stereotypes. The syntax is PROFILE.NAME, where PROFILE is the full path of the Profile that owns the Stereotype.") Set<String> stereotypes,
@QueryParam("taggedValue")
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes)
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes,
@QueryParam("excludeInvalid")
@DefaultValue("false")
@ApiParam("If true, exclude invalid entities and return valid entities only. If false, the endpoint will return an error if there are any invalid entities.") boolean excludeInvalid)
{
return executeWithLogging(
"getting [from] entities in review " + reviewId + " for project " + projectId,
() -> getEntities(this.entityApi.getReviewFromEntityAccessContext(projectId, reviewId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes)
() -> getEntities(this.entityApi.getReviewFromEntityAccessContext(projectId, reviewId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes, excludeInvalid)
);
}

Expand All @@ -87,11 +90,14 @@ public List<Entity> getReviewToEntities(@PathParam("projectId") String projectId
@QueryParam("stereotype")
@ApiParam("Only include entities with one of these stereotypes. The syntax is PROFILE.NAME, where PROFILE is the full path of the Profile that owns the Stereotype.") Set<String> stereotypes,
@QueryParam("taggedValue")
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes)
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes,
@QueryParam("excludeInvalid")
@DefaultValue("false")
@ApiParam("If true, exclude invalid entities and return valid entities only. If false, the endpoint will return an error if there are any invalid entities.") boolean excludeInvalid)
{
return executeWithLogging(
"getting [to] entities in review " + reviewId + " for project " + projectId,
() -> getEntities(this.entityApi.getReviewToEntityAccessContext(projectId, reviewId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes)
() -> getEntities(this.entityApi.getReviewToEntityAccessContext(projectId, reviewId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes, excludeInvalid)
);
}

Expand Down
7 changes: 5 additions & 2 deletions ...a/org/finos/legend/sdlc/server/resources/ConflictResolutionWorkspaceEntitiesResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,14 @@ public List<Entity> getAllEntities(@PathParam("projectId") String projectId,
@QueryParam("stereotype")
@ApiParam("Only include entities with one of these stereotypes. The syntax is PROFILE.NAME, where PROFILE is the full path of the Profile that owns the Stereotype.") Set<String> stereotypes,
@QueryParam("taggedValue")
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes)
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes,
@QueryParam("excludeInvalid")
@DefaultValue("false")
@ApiParam("If true, exclude invalid entities and return valid entities only. If false, the endpoint will return an error if there are any invalid entities.") boolean excludeInvalid)
{
return executeWithLogging(
"getting entities in user workspace with conflict resolution " + workspaceId + " for project " + projectId,
() -> getEntities(this.entityApi.getUserWorkspaceWithConflictResolutionEntityAccessContext(projectId, workspaceId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes)
() -> getEntities(this.entityApi.getUserWorkspaceWithConflictResolutionEntityAccessContext(projectId, workspaceId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes, excludeInvalid)
);
}

Expand Down
13 changes: 9 additions & 4 deletions ...nos/legend/sdlc/server/resources/ConflictResolutionWorkspaceRevisionEntitiesResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ public ConflictResolutionWorkspaceRevisionEntitiesResource(EntityApi entityApi)
@ApiOperation("Get entities of the user workspace with conflict resolution at the revision")
public List<Entity> getAllEntities(@PathParam("projectId") String projectId,
@PathParam("workspaceId") String workspaceId,
@PathParam("revisionId") @ApiParam("Including aliases: head, latest, current, base") String revisionId,
@PathParam("revisionId")
@ApiParam("Including aliases: head, latest, current, base") String revisionId,
@QueryParam("classifierPath")
@ApiParam("Only include entities with one of these classifier paths.") Set<String> classifierPaths,
@QueryParam("package")
Expand All @@ -63,11 +64,14 @@ public List<Entity> getAllEntities(@PathParam("projectId") String projectId,
@QueryParam("stereotype")
@ApiParam("Only include entities with one of these stereotypes. The syntax is PROFILE.NAME, where PROFILE is the full path of the Profile that owns the Stereotype.") Set<String> stereotypes,
@QueryParam("taggedValue")
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes)
@ApiParam("Only include entities with a matching tagged value. The syntax is PROFILE.NAME/REGEX, where PROFILE is the full path of the Profile that owns the Tag, NAME is the name of the Tag, and REGEX is a regular expression to match against the value.") List<String> taggedValueRegexes,
@QueryParam("excludeInvalid")
@DefaultValue("false")
@ApiParam("If true, exclude invalid entities and return valid entities only. If false, the endpoint will return an error if there are any invalid entities.") boolean excludeInvalid)
{
return executeWithLogging(
"getting entities in revision " + revisionId + " of user workspace with conflict resolution " + workspaceId + " for project " + projectId,
() -> getEntities(this.entityApi.getUserWorkspaceWithConflictResolutionRevisionEntityAccessContext(projectId, workspaceId, revisionId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes)
() -> getEntities(this.entityApi.getUserWorkspaceWithConflictResolutionRevisionEntityAccessContext(projectId, workspaceId, revisionId), classifierPaths, packages, includeSubPackages, nameRegex, stereotypes, taggedValueRegexes, excludeInvalid)
);
}

Expand All @@ -76,7 +80,8 @@ public List<Entity> getAllEntities(@PathParam("projectId") String projectId,
@ApiOperation("Get an entity of the user workspace with conflict resolution at the revision by its path")
public Entity getEntityByPath(@PathParam("projectId") String projectId,
@PathParam("workspaceId") String workspaceId,
@PathParam("revisionId") @ApiParam("Including aliases: head, latest, current, base") String revisionId,
@PathParam("revisionId")
@ApiParam("Including aliases: head, latest, current, base") String revisionId,
@PathParam("path") String path)
{
return executeWithLogging(
Expand Down
Loading

0 comments on commit 3da555a

Please sign in to comment.