Skip to content

Commit

Permalink
Merge pull request #4 from gs-gunjan/patch-6
Browse files Browse the repository at this point in the history 
Patch 6
  • Loading branch information
@gs-gunjan
gs-gunjan authored Aug 9, 2023
2 parents 7555b4d + 4d55eb4 commit 7247d7d
Show file tree
Hide file tree
Showing 26 changed files with 46 additions and 33 deletions.
2 changes: 1 addition & 1 deletion legend-depot-artifacts-purge/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-artifacts-refresh/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-artifacts-repository-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-artifacts-repository-maven-impl/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-core-authorisation/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-core-http/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
10 changes: 10 additions & 0 deletions ...-depot-core-http/src/main/java/org/finos/legend/depot/tracing/resources/BaseResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import javax.ws.rs.core.CacheControl;
import javax.ws.rs.core.EntityTag;
import javax.ws.rs.core.Request;
import javax.ws.rs.core.Response;
Expand Down Expand Up @@ -105,11 +106,20 @@ protected <T> Response handle(String resourceAPIMetricName, String label, Suppli
{
return Response.noContent().status(Response.Status.NOT_MODIFIED).build();
}
CacheControl cc = new CacheControl();
Response.ResponseBuilder responseBuilder = Response.ok(handle(resourceAPIMetricName, label, supplier));
if (serverTag != null)
{
responseBuilder.tag(serverTag);
cc.setMustRevalidate(true);
}
else
{
cc.setNoCache(true);
cc.setNoStore(true);
}

responseBuilder.cacheControl(cc);
return responseBuilder.build();
}

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-core-schedules/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-core-services/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-core-tracing/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-model/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-pure-model-context/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-server/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-servers-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-store-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-store-metrics/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
4 changes: 3 additions & 1 deletion legend-depot-store-mongo/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,6 @@ Currently, with Mongo implementation there is limit on how much nesting of an ob
In order to store such entities and also make the storage of entities scalable to other databases, we are going to be storing the content as string.
With this change we have also introduced three kinds of entity storage which will further help the cause of scalability.
### Steps to follow:
1. Use the API /migrations/migrateToStoredEntityData, this will update the entities collection to be identified as one form of entity data.
1. Use the API /migrations/migrateToStoredEntityData, this will update the entities collection to be identified as one form of entity data.
2. Once done, using the /indexes (create indexes if absent) you can create the index for this collection.
3. Delete the old indexes of the entities collection.
2 changes: 1 addition & 1 deletion legend-depot-store-mongo/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
6 changes: 3 additions & 3 deletions ...ongo/src/main/java/org/finos/legend/depot/store/mongo/entities/AbstractEntitiesMongo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ public List<T> getStoredEntities(String groupId, String artifactId, String versi

public List<Entity> getAllEntities(String groupId, String artifactId, String versionId)
{
return find(getArtifactAndVersionVersionedFilter(groupId, artifactId, versionId)).stream().map(this::resolvedToEntityDefinition).collect(Collectors.toList());
return find(getArtifactAndVersionVersionedFilter(groupId, artifactId, versionId)).parallelStream().map(this::resolvedToEntityDefinition).collect(Collectors.toList());
}

public List<Entity> getEntitiesByPackage(String groupId, String artifactId, String versionId, String packageName, Set<String> classifierPaths, boolean includeSubPackages)
Expand All @@ -161,7 +161,7 @@ public List<Entity> getEntitiesByPackage(String groupId, String artifactId, Stri
{
filter = and(filter, eq(ENTITY_PACKAGE, packageName));
}
Stream<Entity> entities = find(filter).stream().map(this::resolvedToEntityDefinition);
Stream<Entity> entities = find(filter).parallelStream().map(this::resolvedToEntityDefinition);
if (classifierPaths != null && !classifierPaths.isEmpty())
{
entities = entities.filter(entity -> classifierPaths.contains(entity.getClassifierPath()));
Expand All @@ -181,7 +181,7 @@ public FindIterable findLatestEntitiesByClassifier(String classifier)

public List<Entity> findEntitiesByClassifier(String groupId, String artifactId, String versionId, String classifier)
{
return find(and(getArtifactAndVersionVersionedFilter(groupId, artifactId, versionId), eq(ENTITY_CLASSIFIER_PATH, classifier))).stream().map(this::resolvedToEntityDefinition).collect(Collectors.toList());
return find(and(getArtifactAndVersionVersionedFilter(groupId, artifactId, versionId), eq(ENTITY_CLASSIFIER_PATH, classifier))).parallelStream().map(this::resolvedToEntityDefinition).collect(Collectors.toList());
}

public long delete(String groupId, String artifactId, String versionId)
Expand Down
8 changes: 4 additions & 4 deletions ...-store-mongo/src/main/java/org/finos/legend/depot/store/mongo/entities/EntitiesMongo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,9 @@ public EntitiesMongo(@Named("mongoDatabase") MongoDatabase databaseProvider, Cla
public static List<IndexModel> buildIndexes()
{
return Arrays.asList(buildIndex("groupId-artifactId-versionId", GROUP_ID, ARTIFACT_ID, VERSION_ID),
buildIndex("groupId-artifactId-versionId-entityPath", true, GROUP_ID, ARTIFACT_ID, VERSION_ID, ENTITY_PATH),
buildIndex("groupId-artifactId-versionId-package", GROUP_ID, ARTIFACT_ID, VERSION_ID, ENTITY_PACKAGE),
buildIndex("entity-classifier", ENTITY_CLASSIFIER_PATH)
buildIndex("groupId-artifactId-versionId-entityAttributes-path", true, GROUP_ID, ARTIFACT_ID, VERSION_ID, ENTITY_PATH),
buildIndex("groupId-artifactId-versionId-entityAttributes-package", GROUP_ID, ARTIFACT_ID, VERSION_ID, ENTITY_PACKAGE),
buildIndex("entityAttributes-classifier", ENTITY_CLASSIFIER_PATH)
);
}

Expand Down Expand Up @@ -119,7 +119,7 @@ protected void validateNewData(T entity)
public List<T> createOrUpdate(String groupId, String artifactId, String versionId, List<Entity> entityDefinitions)
{
List<StoredEntity> versionedEntities = new ArrayList<>();
entityDefinitions.forEach(item ->
entityDefinitions.parallelStream().forEach(item ->
{
StoredEntity storedEntity = new StoredEntityStringData(groupId, artifactId, versionId);
getCollection().updateOne(getEntityPathFilter(groupId, artifactId, versionId, item.getPath()), combineDocument((T) storedEntity, item, ENTITY_TYPE_STRING_DATA), INSERT_IF_ABSENT);
Expand Down
2 changes: 1 addition & 1 deletion ...ain/java/org/finos/legend/depot/store/mongo/versionedEntities/VersionedEntitiesMongo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ protected MongoCollection getCollection()
public List<StoredVersionedEntity> createOrUpdate(String groupId, String artifactId, String versionId, List<Entity> entityDefinitions)
{
List<StoredVersionedEntity> versionedEntities = new ArrayList<>();
entityDefinitions.forEach(item ->
entityDefinitions.parallelStream().forEach(item ->
{
StoredVersionedEntity storedEntity = new StoredVersionedEntityStringData(groupId, artifactId, versionId);
getCollection().updateOne(getEntityPathFilter(groupId, artifactId, versionId, item.getPath()), combineDocument(storedEntity, item, VERSIONED_ENTITY_TYPE_STRING_DATA), INSERT_IF_ABSENT);
Expand Down
2 changes: 1 addition & 1 deletion legend-depot-store-notifications-queue/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-store-notifications/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<artifactId>legend-depot</artifactId>
<groupId>org.finos.legend.depot</groupId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-store-server/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion legend-depot-test-reports/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
9 changes: 5 additions & 4 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
<name>Legend Depot</name>
<groupId>org.finos.legend.depot</groupId>
<artifactId>legend-depot</artifactId>
<version>1.21.6-SNAPSHOT</version>
<version>1.23.1-SNAPSHOT</version>
<packaging>pom</packaging>

<modules>
Expand Down Expand Up @@ -78,8 +78,8 @@

<!-- Legend dependency versions -->
<legend.shared.version>0.23.8</legend.shared.version>
<legend.engine.version>4.18.2</legend.engine.version>
<legend.sdlc.version>0.136.4</legend.sdlc.version>
<legend.engine.version>4.19.0</legend.engine.version>
<legend.sdlc.version>0.137.0</legend.sdlc.version>


<!-- Dependency versions -->
Expand Down Expand Up @@ -149,7 +149,7 @@
<maven.plugin.plugin.version>3.6.0</maven.plugin.plugin.version>
<maven.enforcer.plugin.version>3.0.0-M1</maven.enforcer.plugin.version>
<maven.jar.plugin.version>3.1.2</maven.jar.plugin.version>
<nexus.staging.maven.plugin.version>1.6.8</nexus.staging.maven.plugin.version>
<nexus.staging.maven.plugin.version>1.6.13</nexus.staging.maven.plugin.version>

<git.commit.plugin.version>2.2.6</git.commit.plugin.version>
<versions.maven.plugin.version>2.7</versions.maven.plugin.version>
Expand Down Expand Up @@ -396,6 +396,7 @@
<serverId>ossrh</serverId>
<nexusUrl>https://s01.oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>true</autoReleaseAfterClose>
<stagingProgressTimeoutMinutes>10</stagingProgressTimeoutMinutes>
</configuration>
</plugin>
<plugin>
Expand Down

0 comments on commit 7247d7d

Please sign in to comment.