Merge branch 'finos:master' into patch-3

gs-gunjan · Aug 10, 2023 · 6336999 · 6336999
2 parents 75a76d5 + 240f8ad
commit 6336999
Show file tree

Hide file tree

Showing 152 changed files with 1,286 additions and 722 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -67,11 +67,11 @@ jobs:
           theme: dark
 
       - name: Build + Test
-        if: (github.repository != 'finos/legend-depot') || (github.ref != 'refs/heads/master')
+        if: (github.repository != 'finos/legend-depot') || (github.ref != 'refs/heads/master') || (github.ref != 'refs/heads/integration')
         run: mvn install javadoc:javadoc
 
       - name: Build + Test + Sonar + Maven Deploy
-        if: (github.repository == 'finos/legend-depot') && (github.ref == 'refs/heads/master')
+        if: (github.repository == 'finos/legend-depot') && ((github.ref == 'refs/heads/master') || (github.ref == 'refs/heads/integration'))
         env:
           DOCKER_USERNAME: finos
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

diff --git a/.github/workflows/clean-after-failed-release.yml b/.github/workflows/clean-after-failed-release.yml
@@ -28,6 +28,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          ref: 'master'
           token: ${{ secrets.FINOS_GITHUB_TOKEN }}
 
       - name: Configure git

diff --git a/.github/workflows/legend-stack-release.yml b/.github/workflows/legend-stack-release.yml
@@ -33,6 +33,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
+          ref: 'master'
           token: ${{ secrets.FINOS_GITHUB_TOKEN }}
 
       - name: Configure git

diff --git a/.github/workflows/ossrh-close-staging-repo.yml b/.github/workflows/ossrh-close-staging-repo.yml
@@ -40,6 +40,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
+          ref: 'master'
           token: ${{ secrets.FINOS_GITHUB_TOKEN }}
 
       - name: Configure git

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -36,6 +36,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
+          ref: 'master'
           token: ${{ secrets.FINOS_GITHUB_TOKEN }}
 
       - name: Configure git

diff --git a/legend-depot-artifacts-purge/pom.xml b/legend-depot-artifacts-purge/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>legend-depot</artifactId>
         <groupId>org.finos.legend.depot</groupId>
-        <version>1.24.1-SNAPSHOT</version>
+        <version>1.25.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -31,23 +31,36 @@
         <!-- DEPOT -->
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-artifacts-refresh</artifactId>
+            <artifactId>legend-depot-model</artifactId>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-core-authorisation</artifactId>
+            <artifactId>legend-depot-core-services</artifactId>
         </dependency>
+        <!--TODO: remove this dependency -->
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-core-services</artifactId>
+            <artifactId>legend-depot-artifacts-refresh</artifactId>
         </dependency>
+
         <dependency>
-            <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-store-metrics</artifactId>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+        </dependency>
+
+        <!-- testing-->
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-core-schedules</artifactId>
+            <artifactId>legend-depot-generations-services</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
@@ -56,28 +69,22 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
-
-        <!-- DEPOT -->
-
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-generations-store-mongo</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.google.inject</groupId>
-            <artifactId>guice</artifactId>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-generations-store-mongo</artifactId>
+            <classifier>tests</classifier>
+            <type>test-jar</type>
+            <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
+            <groupId>de.bwaldvogel</groupId>
+            <artifactId>mongo-java-server</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
               <groupId>${junit.groupId}</groupId>
@@ -89,10 +96,5 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>de.bwaldvogel</groupId>
-            <artifactId>mongo-java-server</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>
diff --git a/...artifacts/purge/ArtifactsPurgeModule.java → ...cts/purge/guice/ArtifactsPurgeModule.java b/...artifacts/purge/ArtifactsPurgeModule.java → ...cts/purge/guice/ArtifactsPurgeModule.java
@@ -13,16 +13,16 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.store.artifacts.purge;
+package org.finos.legend.depot.store.artifacts.purge.guice;
 
 import com.google.inject.PrivateModule;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
-import org.finos.legend.depot.schedules.services.SchedulesFactory;
+import org.finos.legend.depot.services.schedules.SchedulesFactory;
 import org.finos.legend.depot.store.artifacts.configuration.ArtifactsRetentionPolicyConfiguration;
 import org.finos.legend.depot.store.artifacts.purge.api.ArtifactsPurgeService;
-import org.finos.legend.depot.store.artifacts.purge.resources.ArtifactsPurgeResource;
+import org.finos.legend.depot.store.resources.artifacts.ArtifactsPurgeResource;
 import org.finos.legend.depot.store.artifacts.purge.services.ArtifactsPurgeServiceImpl;
 
 public class ArtifactsPurgeModule extends PrivateModule

diff --git a/...java/org/finos/legend/depot/store/artifacts/purge/services/ArtifactsPurgeServiceImpl.java b/...java/org/finos/legend/depot/store/artifacts/purge/services/ArtifactsPurgeServiceImpl.java
@@ -15,7 +15,7 @@
 
 package org.finos.legend.depot.store.artifacts.purge.services;
 
-import org.finos.legend.depot.artifacts.repository.domain.ArtifactType;
+import org.finos.legend.depot.store.artifacts.repository.domain.ArtifactType;
 import org.finos.legend.depot.domain.VersionedData;
 import org.finos.legend.depot.domain.api.MetadataEventResponse;
 import org.finos.legend.depot.domain.project.ProjectVersion;

diff --git a/...rge/resources/ArtifactsPurgeResource.java → ...ces/artifacts/ArtifactsPurgeResource.java b/...rge/resources/ArtifactsPurgeResource.java → ...ces/artifacts/ArtifactsPurgeResource.java
@@ -13,7 +13,7 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.store.artifacts.purge.resources;
+package org.finos.legend.depot.store.resources.artifacts;
 
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;

diff --git a/...s/services/TestArtifactsPurgeService.java → ...e/services/TestArtifactsPurgeService.java b/...s/services/TestArtifactsPurgeService.java → ...e/services/TestArtifactsPurgeService.java
@@ -13,10 +13,10 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.store.artifacts.services;
+package org.finos.legend.depot.store.artifacts.purge.services;
 
-import org.finos.legend.depot.artifacts.repository.api.ArtifactRepository;
-import org.finos.legend.depot.artifacts.repository.domain.ArtifactType;
+import org.finos.legend.depot.store.artifacts.repository.api.ArtifactRepository;
+import org.finos.legend.depot.store.artifacts.repository.domain.ArtifactType;
 import org.finos.legend.depot.domain.version.VersionMismatch;
 import org.finos.legend.depot.domain.DatesHandler;
 import org.finos.legend.depot.domain.api.MetadataEventResponse;
@@ -26,16 +26,16 @@
 import org.finos.legend.depot.services.api.entities.ManageEntitiesService;
 import org.finos.legend.depot.services.api.projects.ManageProjectsService;
 import org.finos.legend.depot.services.entities.ManageEntitiesServiceImpl;
-import org.finos.legend.depot.services.generation.file.ManageFileGenerationsServiceImpl;
+import org.finos.legend.depot.services.generations.impl.ManageFileGenerationsServiceImpl;
 import org.finos.legend.depot.services.projects.ManageProjectsServiceImpl;
 import org.finos.legend.depot.services.projects.configuration.ProjectsConfiguration;
 import org.finos.legend.depot.store.admin.domain.metrics.VersionQueryMetric;
 import org.finos.legend.depot.store.api.entities.UpdateEntities;
-import org.finos.legend.depot.store.api.generation.file.UpdateFileGenerations;
+import org.finos.legend.depot.store.api.generations.UpdateFileGenerations;
 import org.finos.legend.depot.store.api.projects.UpdateProjects;
 import org.finos.legend.depot.store.api.projects.UpdateProjectsVersions;
 import org.finos.legend.depot.store.artifacts.purge.api.ArtifactsPurgeService;
-import org.finos.legend.depot.store.artifacts.purge.services.ArtifactsPurgeServiceImpl;
+import org.finos.legend.depot.store.artifacts.services.ProjectArtifactHandlerFactory;
 import org.finos.legend.depot.store.artifacts.services.entities.EntitiesHandlerImpl;
 import org.finos.legend.depot.store.artifacts.services.entities.EntityProvider;
 import org.finos.legend.depot.store.artifacts.services.file.FileGenerationHandlerImpl;
@@ -46,10 +46,11 @@
 import org.finos.legend.depot.store.mongo.TestStoreMongo;
 import org.finos.legend.depot.store.metrics.store.mongo.QueryMetricsMongo;
 import org.finos.legend.depot.store.mongo.entities.EntitiesMongo;
-import org.finos.legend.depot.store.mongo.generation.file.FileGenerationsMongo;
+import org.finos.legend.depot.store.mongo.generations.FileGenerationsMongo;
 import org.finos.legend.depot.store.mongo.projects.ProjectsMongo;
 import org.finos.legend.depot.store.mongo.projects.ProjectsVersionsMongo;
 import org.finos.legend.depot.store.notifications.queue.api.Queue;
+import org.finos.legend.depot.store.mongo.generations.TestGenerationsStoreMongo;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -99,7 +100,7 @@ public void setUpData()
         Assert.assertEquals(3, projectsStore.getAll().size());
 
         setUpEntitiesDataFromFile(TestArtifactsPurgeService.class.getClassLoader().getResource("data/entities.json"));
-        setUpFileGenerationFromFile(TestArtifactsPurgeService.class.getClassLoader().getResource("data/generations.json"));
+        TestGenerationsStoreMongo.setUpFileGenerationFromFile(TestArtifactsPurgeService.class.getClassLoader().getResource("data/generations.json"),mongoProvider);
     }
 
 

diff --git a/legend-depot-artifacts-refresh/pom.xml b/legend-depot-artifacts-refresh/pom.xml
@@ -20,7 +20,7 @@
     <parent>
         <artifactId>legend-depot</artifactId>
         <groupId>org.finos.legend.depot</groupId>
-        <version>1.24.1-SNAPSHOT</version>
+        <version>1.25.1-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -35,58 +35,72 @@
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-artifacts-repository-maven-impl</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-artifacts-repository-maven-impl</artifactId>
-            <classifier>tests</classifier>
-            <type>test-jar</type>
-            <scope>test</scope>
+            <artifactId>legend-depot-artifacts-repository-api</artifactId>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
             <artifactId>legend-depot-core-services</artifactId>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
-            <artifactId>legend-depot-store-mongo</artifactId>
-            <classifier>tests</classifier>
-            <type>test-jar</type>
-            <scope>test</scope>
+            <artifactId>legend-depot-generations-api</artifactId>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.depot</groupId>
             <artifactId>legend-depot-store-notifications</artifactId>
         </dependency>
-        <!-- DEPOT -->
 
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
         <!-- SDLC -->
         <dependency>
             <groupId>org.finos.legend.sdlc</groupId>
             <artifactId>legend-sdlc-entity-serialization</artifactId>
         </dependency>
         <!-- SDLC -->
 
+
+
+        <!-- testing -->
+        <dependency>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-generations-services</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-artifacts-repository-maven-impl</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-artifacts-repository-maven-impl</artifactId>
+            <classifier>tests</classifier>
+            <type>test-jar</type>
+            <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-store-mongo</artifactId>
+            <classifier>tests</classifier>
+            <type>test-jar</type>
+            <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.google.inject</groupId>
-            <artifactId>guice</artifactId>
+            <groupId>org.finos.legend.depot</groupId>
+            <artifactId>legend-depot-generations-store-mongo</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
+            <groupId>de.bwaldvogel</groupId>
+            <artifactId>mongo-java-server</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>${junit.groupId}</groupId>
@@ -98,10 +112,7 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>de.bwaldvogel</groupId>
-            <artifactId>mongo-java-server</artifactId>
-            <scope>test</scope>
-        </dependency>
+
+
     </dependencies>
 </project>
diff --git a/...e/artifacts/ArtifactLoadingException.java → ...tifacts/api/ArtifactLoadingException.java b/...e/artifacts/ArtifactLoadingException.java → ...tifacts/api/ArtifactLoadingException.java
@@ -13,7 +13,7 @@
 //  limitations under the License.
 //
 
-package org.finos.legend.depot.store.artifacts;
+package org.finos.legend.depot.store.artifacts.api;
 
 public class ArtifactLoadingException extends RuntimeException
 {

diff --git a/...-refresh/src/main/java/org/finos/legend/depot/store/artifacts/api/ArtifactsExtractor.java b/...-refresh/src/main/java/org/finos/legend/depot/store/artifacts/api/ArtifactsExtractor.java
@@ -15,7 +15,7 @@
 
 package org.finos.legend.depot.store.artifacts.api;
 
-import org.finos.legend.depot.artifacts.repository.domain.ArtifactType;
+import org.finos.legend.depot.store.artifacts.repository.domain.ArtifactType;
 
 import java.io.File;
 import java.util.List;

diff --git a/...esh/src/main/java/org/finos/legend/depot/store/artifacts/api/ProjectArtifactsHandler.java b/...esh/src/main/java/org/finos/legend/depot/store/artifacts/api/ProjectArtifactsHandler.java
@@ -16,15 +16,14 @@
 package org.finos.legend.depot.store.artifacts.api;
 
 import org.finos.legend.depot.domain.api.MetadataEventResponse;
-import org.finos.legend.depot.domain.project.StoreProjectData;
 
 import java.io.File;
 import java.util.List;
 
 public interface ProjectArtifactsHandler
 {
 
-    MetadataEventResponse refreshProjectVersionArtifacts(StoreProjectData projectData, String versionId, List<File> files);
+    MetadataEventResponse refreshProjectVersionArtifacts(String groupId,String artifactId, String versionId, List<File> files);
 
     void delete(String groupId, String artifactId, String versionId);
 }