Merge branch 'BlueBrain:master' into 3890-batch-sink

BlueBrain · Jul 28, 2023 · a6dd24d · a6dd24d
2 parents bd6f044 + 7b9a9c8
commit a6dd24d
Show file tree

Hide file tree

Showing 122 changed files with 2,470 additions and 1,839 deletions.
diff --git a/.github/workflows/ci-delta-app.yml b/.github/workflows/ci-delta-app.yml
@@ -0,0 +1,41 @@
+name: Delta App
+on:
+  pull_request:
+    paths:
+      - 'delta/**'
+      - 'build.sbt'
+      - 'project/**'
+      - '.github/workflows/ci-delta-app.yml'
+jobs:
+  static-analysis:
+    if: github.event_name == 'pull_request'
+    runs-on: it
+    timeout-minutes: 10
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Static analysis
+        run: |
+          sbt -Dsbt.color=always -Dsbt.supershell=false \
+            clean \
+            app-static-analysis
+  unit-tests:
+    if: github.event_name == 'pull_request'
+    runs-on: it
+    timeout-minutes: 20
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Unit tests
+        run: |
+          sbt -Dsbt.color=always -Dsbt.supershell=false \
+            clean \
+            app-unit-tests-with-coverage
diff --git a/.github/workflows/ci-delta-core.yml b/.github/workflows/ci-delta-core.yml
@@ -2,7 +2,12 @@ name: Delta Core
 on:
   pull_request:
     paths:
-      - 'delta/**'
+      - 'delta/kernel/**'
+      - 'delta/plugins/**'
+      - 'delta/rdf/**'
+      - 'delta/sdk/**'
+      - 'delta/sourcing-psql/**'
+      - 'delta/testkit/**'
       - 'build.sbt'
       - 'project/**'
       - '.github/workflows/ci-delta-core.yml'
@@ -18,16 +23,12 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Static Analysis
+      - name: Static analysis
         run: |
           sbt -Dsbt.color=always -Dsbt.supershell=false \
-            "project delta" \
             clean \
-            scalafmtCheck \
-            Test/scalafmtCheck \
-            scalafmtSbtCheck \
-            scapegoat \
-            doc
+            core-static-analysis
+
   unit-tests:
     if: github.event_name == 'pull_request'
     runs-on: it
@@ -39,20 +40,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Tests
+      - name: Unit tests
         run: |
           sbt -Dsbt.color=always -Dsbt.supershell=false \
             clean \
-            coverage \
-            app/test \
-            kernel/test \
-            rdf/test \
-            sdk/test \
-            sourcingPsql/test \
-            testkit/test \
-            app/coverageReport \
-            kernel/coverageReport \
-            rdf/coverageReport \
-            sdk/coverageReport \
-            sourcingPsql/coverageReport \
-            testkit/coverageReport
+            core-unit-tests-with-coverage
diff --git a/.github/workflows/ci-delta-plugins.yml b/.github/workflows/ci-delta-plugins.yml
@@ -2,11 +2,32 @@ name: Delta Plugins
 on:
   pull_request:
     paths:
-      - 'delta/**'
+      - 'delta/kernel/**'
+      - 'delta/plugins/**'
+      - 'delta/rdf/**'
+      - 'delta/sdk/**'
+      - 'delta/sourcing-psql/**'
+      - 'delta/testkit/**'
       - 'build.sbt'
       - 'project/**'
       - '.github/workflows/ci-delta-plugins.yml'
 jobs:
+  static-analysis:
+    if: github.event_name == 'pull_request'
+    runs-on: it
+    timeout-minutes: 10
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Static analysis
+        run: |
+          sbt -Dsbt.color=always -Dsbt.supershell=false \
+            clean \
+            plugins-static-analysis
   unit-tests:
     if: github.event_name == 'pull_request'
     runs-on: it
@@ -18,10 +39,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Tests
+      - name: Unit tests
         run: |
           sbt -Dsbt.color=always -Dsbt.supershell=false \
             clean \
-            coverage \
-            plugins/test \
-            plugins/coverageReport
+            plugins-unit-tests-with-coverage
diff --git a/build.sbt b/build.sbt
@@ -1005,10 +1005,54 @@ Global / excludeLintKeys        += docs / paradoxRoots
 Global / excludeLintKeys        += docs / Paradox / paradoxNavigationDepth
 Global / concurrentRestrictions += Tags.limit(Tags.Test, 1)
 
-addCommandAlias("review", ";clean;scalafmtCheck;test:scalafmtCheck;scalafmtSbtCheck;coverage;scapegoat;test;coverageReport;coverageAggregate")
+addCommandAlias("review",
+  s"""
+     |;clean
+     |;scalafmtCheck
+     |;test:scalafmtCheck
+     |;scalafmtSbtCheck
+     |;coverage
+     |;scapegoat
+     |;test
+     |;coverageReport
+     |;coverageAggregate
+     |""".stripMargin
+)
 addCommandAlias(
   "deltaReview",
-  ";delta/clean;delta/scalafmtCheck;delta/test:scalafmtCheck;scalafmtSbtCheck;coverage;delta/scapegoat;delta/test;delta/coverageReport;delta/coverageAggregate"
+  """
+     |;delta/clean
+     |;delta/scalafmtCheck
+     |;delta/test:scalafmtCheck
+     |;scalafmtSbtCheck;coverage
+     |;delta/scapegoat
+     |;delta/test
+     |;delta/coverageReport
+     |;delta/coverageAggregate
+     |""".stripMargin
 )
 addCommandAlias("build-docs", ";docs/clean;docs/makeSite")
 addCommandAlias("preview-docs", ";docs/clean;docs/previewSite")
+
+val coreModules = List("kernel", "rdf", "sdk", "sourcingPsql", "testkit")
+def staticAnalysisCommandForModule(module: String) =
+  s"""
+    |;$module/scalafmtCheck
+    |;$module/Test/scalafmtCheck
+    |;$module/scalafmtSbtCheck
+    |;$module/scapegoat
+    |;$module/doc
+    |""".stripMargin
+
+addCommandAlias("core-static-analysis", coreModules.map(staticAnalysisCommandForModule).mkString)
+addCommandAlias("app-static-analysis", staticAnalysisCommandForModule("app"))
+addCommandAlias("plugins-static-analysis", staticAnalysisCommandForModule("plugins"))
+
+def unitTestsWithCoverageCommandsForModules(modules: List[String]) = {
+  ";coverage" +
+    modules.map(module => s";$module/test").mkString +
+    modules.map(module => s";$module/coverageReport").mkString
+}
+addCommandAlias("core-unit-tests-with-coverage", unitTestsWithCoverageCommandsForModules(coreModules))
+addCommandAlias("app-unit-tests-with-coverage", unitTestsWithCoverageCommandsForModules(List("app")))
+addCommandAlias("plugins-unit-tests-with-coverage", unitTestsWithCoverageCommandsForModules(List("plugins")))
diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala
@@ -64,12 +64,13 @@ class ResourcesRoutesSpec extends BaseRouteSpec {
   private val myId2                       = nxv + "myid2" // Resource created against schema1 with id present on the payload
   private val myId3                       = nxv + "myid3" // Resource created against no schema with id passed and present on the payload
   private val myId4                       = nxv + "myid4" // Resource created against schema1 with id passed and present on the payload
+  private val myId5                       = nxv + "myid5" // Resource created against schema1 with id passed and present on the payload
   private val myIdEncoded                 = UrlUtils.encode(myId.toString)
   private val myId2Encoded                = UrlUtils.encode(myId2.toString)
   private val payload                     = jsonContentOf("resources/resource.json", "id" -> myId)
   private val payloadWithBlankId          = jsonContentOf("resources/resource.json", "id" -> "")
   private val payloadWithUnderscoreFields =
-    jsonContentOf("resources/resource-with-underscore-fields.json", "id" -> myId)
+    jsonContentOf("resources/resource-with-underscore-fields.json", "id" -> myId5)
   private val payloadWithMetadata         = jsonContentOf("resources/resource-with-metadata.json", "id" -> myId)
 
   private val aclCheck = AclSimpleCheck().accepted
@@ -195,10 +196,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec {
       val lenientDecodingRoutes = routesWithDecodingOption(DecodingOption.Lenient)
 
       Post("/v1/resources/myorg/myproject/_/", payloadWithUnderscoreFields.toEntity) ~> lenientDecodingRoutes ~> check {
-        response.status shouldEqual StatusCodes.BadRequest
-        response.asJson shouldEqual jsonContentOf(
-          "/resources/errors/underscore-fields.json"
-        )
+        response.status shouldEqual StatusCodes.Created
       }
     }
 

diff --git a/.../main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphPluginModule.scala b/.../main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphPluginModule.scala
@@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.config.BlazegraphViewsCo
 import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.indexing.BlazegraphCoordinator
 import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewRejection.ProjectContextRejection
 import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{contexts, schema => viewsSchemaId, BlazegraphView, BlazegraphViewEvent}
-import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.routes.BlazegraphViewsRoutes
+import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.routes.{BlazegraphViewsIndexingRoutes, BlazegraphViewsRoutes, BlazegraphViewsRoutesHandler}
 import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.slowqueries.{BlazegraphSlowQueryDeleter, BlazegraphSlowQueryLogger, BlazegraphSlowQueryStore}
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
@@ -178,8 +178,6 @@ class BlazegraphPluginModule(priority: Int) extends ModuleDef {
         identities: Identities,
         aclCheck: AclCheck,
         views: BlazegraphViews,
-        projections: Projections,
-        projectionErrors: ProjectionErrors,
         viewsQuery: BlazegraphViewsQuery,
         schemeDirectives: DeltaSchemeDirectives,
         indexingAction: IndexingAction @Id("aggregate"),
@@ -196,8 +194,6 @@ class BlazegraphPluginModule(priority: Int) extends ModuleDef {
         viewsQuery,
         identities,
         aclCheck,
-        projections,
-        projectionErrors,
         schemeDirectives,
         indexingAction(_, _, _)(shift, cr)
       )(
@@ -210,6 +206,36 @@ class BlazegraphPluginModule(priority: Int) extends ModuleDef {
       )
   }
 
+  make[BlazegraphViewsIndexingRoutes].from {
+    (
+        identities: Identities,
+        aclCheck: AclCheck,
+        views: BlazegraphViews,
+        projections: Projections,
+        projectionErrors: ProjectionErrors,
+        schemeDirectives: DeltaSchemeDirectives,
+        baseUri: BaseUri,
+        cfg: BlazegraphViewsConfig,
+        s: Scheduler,
+        cr: RemoteContextResolution @Id("aggregate"),
+        ordering: JsonKeyOrdering
+    ) =>
+      new BlazegraphViewsIndexingRoutes(
+        views.fetchIndexingView(_, _),
+        identities,
+        aclCheck,
+        projections,
+        projectionErrors,
+        schemeDirectives
+      )(
+        baseUri,
+        s,
+        cr,
+        ordering,
+        cfg.pagination
+      )
+  }
+
   make[BlazegraphScopeInitialization].from {
     (views: BlazegraphViews, serviceAccount: ServiceAccount, config: BlazegraphViewsConfig) =>
       new BlazegraphScopeInitialization(views, serviceAccount, config.defaults)
@@ -240,8 +266,22 @@ class BlazegraphPluginModule(priority: Int) extends ModuleDef {
 
   many[ApiMappings].add(BlazegraphViews.mappings)
 
-  many[PriorityRoute].add { (route: BlazegraphViewsRoutes) =>
-    PriorityRoute(priority, route.routes, requiresStrictEntity = true)
+  many[PriorityRoute].add {
+    (
+        bg: BlazegraphViewsRoutes,
+        indexing: BlazegraphViewsIndexingRoutes,
+        schemeDirectives: DeltaSchemeDirectives,
+        baseUri: BaseUri
+    ) =>
+      PriorityRoute(
+        priority,
+        BlazegraphViewsRoutesHandler(
+          schemeDirectives,
+          bg.routes,
+          indexing.routes
+        )(baseUri),
+        requiresStrictEntity = true
+      )
   }
 
   many[ServiceDependency].add { (client: BlazegraphClient @Id("blazegraph-indexing-client")) =>

diff --git a/...aph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala b/...aph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala
@@ -293,23 +293,23 @@ final class BlazegraphViews(
     * Return the existing indexing views in a project in a finite stream
     */
   def currentIndexingViews(project: ProjectRef): ElemStream[IndexingViewDef] =
-    log.currentStates(Predicate.Project(project)).evalMapFilter { envelope =>
+    log.currentStates(Scope.Project(project)).evalMapFilter { envelope =>
       Task.pure(toIndexViewDef(envelope))
     }
 
   /**
     * Return all existing indexing views in a finite stream
     */
   def currentIndexingViews: ElemStream[IndexingViewDef] =
-    log.currentStates(Predicate.Root).evalMapFilter { envelope =>
+    log.currentStates(Scope.Root).evalMapFilter { envelope =>
       Task.pure(toIndexViewDef(envelope))
     }
 
   /**
     * Return the indexing views in a non-ending stream
     */
   def indexingViews(start: Offset): ElemStream[IndexingViewDef] =
-    log.states(Predicate.Root, start).evalMapFilter { envelope =>
+    log.states(Scope.Root, start).evalMapFilter { envelope =>
       Task.pure(toIndexViewDef(envelope))
     }
 

diff --git a/...ain/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/indexing/IndexingViewDef.scala b/...ain/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/indexing/IndexingViewDef.scala
@@ -39,7 +39,15 @@ object IndexingViewDef {
       namespace: String,
       indexingRev: Int,
       rev: Int
-  ) extends IndexingViewDef
+  ) extends IndexingViewDef {
+    def projectionMetadata: ProjectionMetadata =
+      ProjectionMetadata(
+        BlazegraphViews.entityType.value,
+        projection,
+        Some(ref.project),
+        Some(ref.viewId)
+      )
+  }
 
   /**
     * Deprecated view to be cleaned up and removed from the supervisor
@@ -89,22 +97,14 @@ object IndexingViewDef {
       stream: Offset => ElemStream[GraphResource],
       sink: Sink
   ): Task[CompiledProjection] = {
-    val project  = v.ref.project
-    val id       = v.ref.viewId
-    val metadata = ProjectionMetadata(
-      BlazegraphViews.entityType.value,
-      v.projection,
-      Some(project),
-      Some(id)
-    )
 
     val postPipes: Operation = GraphResourceToNTriples
 
     val compiled = for {
       pipes      <- v.pipeChain.traverse(compilePipeChain)
       chain       = pipes.fold(NonEmptyChain.one(postPipes))(NonEmptyChain(_, postPipes))
       projection <- CompiledProjection.compile(
-                      metadata,
+                      v.projectionMetadata,
                       ExecutionStrategy.PersistentSingleNode,
                       Source(stream),
                       chain,
@@ -113,7 +113,7 @@ object IndexingViewDef {
     } yield projection
 
     Task.fromEither(compiled).tapError { e =>
-      Task.delay(logger.error(s"View '$project/$id' could not be compiled.", e))
+      Task.delay(logger.error(s"View '${v.ref}' could not be compiled.", e))
     }
   }
 }