feat(core): migration of core API endpoints to PG backend - part 1

- add DB data conversion utils - improve attributes parsing and validation of values - handle all possible types in attributes values - switch to multi-arch Timescale 2.8.1 / PG 14 image - upgrade to testcontainers 1.17.5
stellio-hub · Jan 29, 2023 · e800b50 · e800b50
1 parent 31d8aba
commit e800b50
Show file tree

Hide file tree

Showing 146 changed files with 10,692 additions and 2,990 deletions.
diff --git a/.env b/.env
@@ -1,21 +1,13 @@
-POSTGRES_PASSWORD=postgres_password
-
-NEO4J_PASSWORD=neo4j_password
-NEO4J_DEFAULT_DATABASE=stellio
-NEO4J_ALLOW_UPGRADE=false
-
+POSTGRES_PASS=stellio_password
+POSTGRES_USER=stellio
 STELLIO_SEARCH_DB_DATABASE=stellio_search
-STELLIO_SEARCH_DB_USER=stellio_search
-STELLIO_SEARCH_DB_PASSWORD=stellio_search_db_password
-
 STELLIO_SUBSCRIPTION_DB_DATABASE=stellio_subscription
-STELLIO_SUBSCRIPTION_DB_USER=stellio_subscription
-STELLIO_SUBSCRIPTION_DB_PASSWORD=stellio_subscription_db_password
+POSTGRES_DBNAME=${STELLIO_SEARCH_DB_DATABASE},${STELLIO_SUBSCRIPTION_DB_DATABASE}
 
 STELLIO_DOCKER_TAG=dev
 
 STELLIO_AUTHENTICATION_ENABLED=false
 
 ENVIRONMENT=docker
 
-APPLICATION_ENTITY_SERVICE_URL=http://entity-service:8082
+APPLICATION_ENTITY_SERVICE_URL=http://search-service:8083
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -33,17 +33,6 @@ pipeline {
                 sh './gradlew build -p api-gateway'
             }
         }
-        stage('Build Entity Service') {
-            when {
-                anyOf {
-                    changeset "entity-service/**"
-                    changeset "shared/**"
-                }
-            }
-            steps {
-                sh './gradlew build -p entity-service'
-            }
-        }
         stage('Build Search Service') {
             when {
                 anyOf {
@@ -96,7 +85,6 @@ pipeline {
                         for (int i = 0; i < currentTags.size(); ++i) {
                             env.CURRENT_TAG = currentTags[i]
                             sh './gradlew jib -Djib.to.image=stellio/stellio-api-gateway:$CURRENT_TAG -Djib.to.auth.username=$EGM_CI_DH_USR -Djib.to.auth.password=$EGM_CI_DH_PSW -p api-gateway'
-                            sh './gradlew jib -Djib.to.image=stellio/stellio-entity-service:$CURRENT_TAG -Djib.to.auth.username=$EGM_CI_DH_USR -Djib.to.auth.password=$EGM_CI_DH_PSW -p entity-service'
                             sh './gradlew jib -Djib.to.image=stellio/stellio-search-service:$CURRENT_TAG -Djib.to.auth.username=$EGM_CI_DH_USR -Djib.to.auth.password=$EGM_CI_DH_PSW -p search-service'
                             sh './gradlew jib -Djib.to.image=stellio/stellio-subscription-service:$CURRENT_TAG -Djib.to.auth.username=$EGM_CI_DH_USR -Djib.to.auth.password=$EGM_CI_DH_PSW -p subscription-service'
                         }
@@ -119,6 +107,8 @@ pipeline {
                     build job: '../DataHub.Int.Launcher'
                 else if (env.BRANCH_NAME == 'develop')
                     build job: '../DataHub.Api-Tests.Launcher'
+                else if (env.BRANCH_NAME == "feature/core-api-migration")
+                    build job: '../DataHub.NGSI-LD-Test-Suite.Launcher'
             }
             slackSend (color: '#36b37e', message: "Success: Stellio on branch ${env.BRANCH_NAME} after ${currentBuild.durationString.replace(' and counting', '')} (<${env.BUILD_URL}|Open>)")
         }

diff --git a/README.md b/README.md
@@ -31,13 +31,12 @@ The NGSI-LD Specification is regularly updated published by ETSI. The latest spe
 -  An Excel file detailing the current compatibility of the  development version of the Stellio Context Broker against the features of the 1.6.1 specification can be downloaded [here](https://docs.google.com/spreadsheets/d/e/2PACX-1vRxOjsDf3lqhwuypJ---pZN2OlqFRl0jyoTV0ewQ1WFnpe7xQary3uxRjunbgJkwQ/pub?output=xlsx)
 
 | :books: [Documentation](https://stellio.rtfd.io/) | :whale: [Docker Hub](https://hub.docker.com/orgs/stellio/repositories) | :dart: [Roadmap](./docs/roadmap.md) |
-| ------------------------------------------------- | ---------------------------------------------------------------------- | ----------------------------------- |
+|---------------------------------------------------|------------------------------------------------------------------------|-------------------------------------|
 
 ## Overview
 
-Stellio is composed of 3 business services:
-* Entity service is in charge of managing the information context, it is backed by a [neo4j](https://neo4j.com) database
-* Search service is in charge of handling the temporal (and geospatial) queries, it is backed by a [TimescaleDB](https://www.timescale.com/) database
+Stellio is composed of 2 business services:
+* Search service is in charge of managing the information context and handling the temporal (and geospatial) queries, it is backed by a [TimescaleDB](https://www.timescale.com/) database
 * Subscription service is in charge of managing subscriptions and subsequent notifications, it is backed by a [TimescaleDB](https://www.timescale.com/) database
 
 It is completed with:
@@ -58,11 +57,10 @@ docker-compose up -d && docker-compose logs -f
 
 It will start all the services composing the Stellio context broker platform and expose them on the following ports:
 * API Gateway: 8080
-* Entity service: 8082
 * Search service: 8083
 * Subscription service: 8084
 
-Please note that the environment and scripts are validated on Ubuntu 19.10 and MacOS. Some errors may occur on other platforms.
+Please note that the environment and scripts are validated on Ubuntu and macOS. Some errors may occur on other platforms.
 
 We also provide an experimental configuration to deploy Stellio in a k8s cluster (only tested in Minikube as of now). For more information, please look at [the README](kubernetes/README.md)
 
@@ -76,14 +74,14 @@ Requirements:
 To develop on a specific service, you can use the provided `docker-compose.yml` file inside each service's directory, for instance:
 
 ```shell script
-cd entity-service
+cd search-service
 docker-compose up -d && docker-compose logs -f
 ```
 
 Then, from the root directory, launch the service:
 
 ```shell script
-./gradlew entity-service:bootRun
+./gradlew search-service:bootRun
 ```
 
 ### Running the tests
@@ -94,7 +92,7 @@ to Spring Boot embedded test support and to the great [TestContainers](https://w
 For instance, you can launch the test suite for entity service with the following command:
 
 ```shell script
-./gradlew entity-service:test
+./gradlew search-service:test
 ```
 
 ### Building the project
@@ -113,7 +111,7 @@ For each service, a self executable jar is produced in the `build/libs` director
 If you want to build only one of the services, you can launch:
 
 ```shell script
-./gradlew entity-service:build
+./gradlew search-service:build
 ```
 
 ### Code quality
@@ -142,19 +140,19 @@ To work locally with a Docker image of a service without publishing it to Docker
 * Build a tar image:
 
 ```shell script
-./gradlew entity-service:jibBuildTar
+./gradlew search-service:jibBuildTar
 ```
 
 * Load the tar image into Docker:
 
 ```shell script
-docker load --input entity-service/build/jib-image.tar
+docker load --input search-service/build/jib-image.tar
 ```
 
 * Run the image:
 
 ```shell script
-docker run stellio/stellio-entity-service:latest
+docker run stellio/stellio-search-service:latest
 ```
 
 ## Usage
@@ -176,17 +174,16 @@ Stellio is licensed under [APL-2.0](https://spdx.org/licenses/Apache-2.0.html).
 
 It mainly makes use of the following libraries and frameworks (dependencies of dependencies have been omitted):
 
-| Library / Framework |	Licence         |
-| ------------------- | --------------- |
-| Spring              | APL v2          |
-| JSON-LD Java        | BSD-3 Clause    |
-| Reactor             | APL v2          |
-| Jackson             |	APL v2          |
-| JUnit               | EPL v2          |
-| Mockk               |	APL v2          |
-| JsonPath            |	APL v2          |
-| WireMock            | APL v2          |
-| Testcontainers      |	MIT             |
-| Neo4j OGM           |	APL v2          |
+| Library / Framework | 	Licence     |
+|---------------------|--------------|
+| Spring              | APL v2       |
+| JSON-LD Java        | BSD-3 Clause |
+| Reactor             | APL v2       |
+| Jackson             | 	APL v2      |
+| JUnit               | EPL v2       |
+| Mockk               | 	APL v2      |
+| JsonPath            | 	APL v2      |
+| WireMock            | APL v2       |
+| Testcontainers      | 	MIT         |
 
 © 2020 - 2022 EGM
diff --git a/api-gateway/src/main/kotlin/com/egm/stellio/apigateway/ApiGatewayApplication.kt b/api-gateway/src/main/kotlin/com/egm/stellio/apigateway/ApiGatewayApplication.kt
@@ -12,9 +12,6 @@ import reactor.netty.transport.logging.AdvancedByteBufFormat
 
 @SpringBootApplication
 class ApiGatewayApplication {
-    @Value("\${application.entity-service.url:entity-service}")
-    private val entityServiceUrl: String = ""
-
     @Value("\${application.search-service.url:search-service}")
     private val searchServiceUrl: String = ""
 
@@ -39,7 +36,7 @@ class ApiGatewayApplication {
                     .filters {
                         it.tokenRelay()
                     }
-                    .uri("http://$entityServiceUrl:8082")
+                    .uri("http://$searchServiceUrl:8083")
             }
             .route { p ->
                 p.path(

diff --git a/api-gateway/src/main/resources/application.yml b/api-gateway/src/main/resources/application.yml
@@ -20,13 +20,6 @@ spring:
   cloud:
     gateway:
       routes:
-        - id: entity_service_actuator
-          uri: http://entity-service:8082
-          predicates:
-            - Path=/entity-service/actuator/**
-          filters:
-            - TokenRelay=
-            - RewritePath=/entity-service/actuator, /actuator
         - id: search_service_actuator
           uri: http://search-service:8083
           predicates:

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -12,7 +12,7 @@ buildscript {
 }
 
 extra["springCloudVersion"] = "2021.0.3"
-extra["testcontainersVersion"] = "1.16.2"
+extra["testcontainersVersion"] = "1.17.5"
 
 plugins {
     java // why did I have to add that ?!

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,42 +21,23 @@ services:
     volumes:
       - ./config/kafka/update_run.sh:/tmp/update_run.sh
     command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'"
-  neo4j:
-    image: neo4j:4.4
-    container_name: stellio-neo4j
-    volumes:
-      - stellio-neo4j-storage:/data
-    environment:
-      - NEO4J_dbms_allow__upgrade=${NEO4J_ALLOW_UPGRADE}
-      - NEO4J_dbms_default__database=${NEO4J_DEFAULT_DATABASE}
-      - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD}
-      - "NEO4J_dbms_security_procedures_unrestricted=apoc.*"
-      - "NEO4J_dbms_security_procedures_allowlist=apoc.*"
-      - NEO4JLABS_PLUGINS=["apoc"]
-    ports:
-      - "7474:7474"
-      - "7687:7687"
-    # following https://github.com/neo4j/neo4j/issues/7203#issuecomment-898511474
-    healthcheck:
-      test: wget -qO- http://localhost:7474 || exit 1
-      interval: 1s
-      timeout: 10s
-      retries: 20
-      start_period: 3s
   postgres:
-    image: stellio/stellio-timescale-postgis:2.3.0-pg13
+    image: stellio/stellio-timescale-postgis:2.8.1-pg14
     container_name: stellio-postgres
     environment:
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - "POSTGRES_MULTIPLE_DATABASES=${STELLIO_SEARCH_DB_DATABASE},${STELLIO_SEARCH_DB_USER},${STELLIO_SEARCH_DB_PASSWORD}: ${STELLIO_SUBSCRIPTION_DB_DATABASE},${STELLIO_SUBSCRIPTION_DB_USER},${STELLIO_SUBSCRIPTION_DB_PASSWORD}"
-      # not sure it is really necessary but it does not break anything ...
-      - PGDATA=/var/lib/postgresql/data/pgdata
+      - DATADIR=/var/lib/postgresql/data
+      - POSTGRES_TEMPLATE_EXTENSIONS=true
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASS=${POSTGRES_PASS}
+      - POSTGRES_DBNAME=${POSTGRES_DBNAME}
+      - POSTGRES_MULTIPLE_EXTENSIONS=postgis,hstore,postgis_topology,postgis_raster,pgrouting,timescaledb,pgcrypto
+      - ACCEPT_TIMESCALE_TUNING=TRUE
     ports:
       - "5432:5432"
     volumes:
       - stellio-postgres-storage:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      test: ["CMD-SHELL", "pg_isready -h localhost -U stellio"]
       interval: 10s
       timeout: 5s
       retries: 20
@@ -68,30 +49,15 @@ services:
       - SPRING_PROFILES_ACTIVE=${ENVIRONMENT}
     ports:
       - "8080:8080"
-  entity-service:
-    container_name: stellio-entity-service
-    image: stellio/stellio-entity-service:${STELLIO_DOCKER_TAG}
-    environment:
-      - SPRING_DATA_NEO4J_PASSWORD=${NEO4J_PASSWORD}
-      - ORG_NEO4J_DRIVER_AUTHENTICATION_PASSWORD=${NEO4J_PASSWORD}
-      - SPRING_PROFILES_ACTIVE=${ENVIRONMENT}
-      - APPLICATION_AUTHENTICATION_ENABLED=${STELLIO_AUTHENTICATION_ENABLED}
-    ports:
-      - "8082:8082"
-    depends_on:
-      neo4j:
-        condition: service_healthy
-      kafka:
-        condition: service_started
   search-service:
     container_name: stellio-search-service
     image: stellio/stellio-search-service:${STELLIO_DOCKER_TAG}
     environment:
       - SPRING_PROFILES_ACTIVE=${ENVIRONMENT}
       - SPRING_R2DBC_URL=r2dbc:postgresql://postgres/${STELLIO_SEARCH_DB_DATABASE}
       - SPRING_FLYWAY_URL=jdbc:postgresql://postgres/${STELLIO_SEARCH_DB_DATABASE}
-      - SPRING_R2DBC_USERNAME=${STELLIO_SEARCH_DB_USER}
-      - SPRING_R2DBC_PASSWORD=${STELLIO_SEARCH_DB_PASSWORD}
+      - SPRING_R2DBC_USERNAME=${POSTGRES_USER}
+      - SPRING_R2DBC_PASSWORD=${POSTGRES_PASS}
       - APPLICATION_AUTHENTICATION_ENABLED=${STELLIO_AUTHENTICATION_ENABLED}
     ports:
       - "8083:8083"
@@ -107,8 +73,8 @@ services:
       - SPRING_PROFILES_ACTIVE=${ENVIRONMENT}
       - SPRING_R2DBC_URL=r2dbc:postgresql://postgres/${STELLIO_SUBSCRIPTION_DB_DATABASE}
       - SPRING_FLYWAY_URL=jdbc:postgresql://postgres/${STELLIO_SUBSCRIPTION_DB_DATABASE}
-      - SPRING_R2DBC_USERNAME=${STELLIO_SUBSCRIPTION_DB_USER}
-      - SPRING_R2DBC_PASSWORD=${STELLIO_SUBSCRIPTION_DB_PASSWORD}
+      - SPRING_R2DBC_USERNAME=${POSTGRES_USER}
+      - SPRING_R2DBC_PASSWORD=${POSTGRES_PASS}
       - APPLICATION_AUTHENTICATION_ENABLED=${STELLIO_AUTHENTICATION_ENABLED}
       - APPLICATION_ENTITY_SERVICE-URL=${APPLICATION_ENTITY_SERVICE_URL}
     ports:

diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityAccessControlHandler.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityAccessControlHandler.kt
@@ -345,13 +345,13 @@ class EntityAccessControlHandler(
             .map {
                 // do a first search without asking for a result in order to get the total count
                 val total = entityService.searchEntities(
-                    QueryParams(types = setOf(it), offset = 0, limit = 0),
+                    QueryParams(types = setOf(it), offset = 0, limit = 0, context = NGSILD_AUTHORIZATION_CONTEXT),
                     sub,
                     NGSILD_AUTHORIZATION_CONTEXT
                 ).first
                 logger.debug("Counted a total of $total entities for type $it")
                 entityService.searchEntities(
-                    QueryParams(types = setOf(it), offset = 0, limit = total),
+                    QueryParams(types = setOf(it), offset = 0, limit = total, context = NGSILD_AUTHORIZATION_CONTEXT),
                     sub,
                     NGSILD_AUTHORIZATION_CONTEXT
                 )
@@ -453,13 +453,18 @@ class EntityAccessControlHandler(
             return ResponseEntity.status(HttpStatus.FORBIDDEN)
                 .body("User is not authorized to sync user referential")
         val total = entityService.searchEntities(
-            QueryParams(attrs = setOf(AUTH_PROP_SAP), offset = 0, limit = 0),
+            QueryParams(attrs = setOf(AUTH_PROP_SAP), offset = 0, limit = 0, context = NGSILD_AUTHORIZATION_CONTEXT),
             sub,
             NGSILD_AUTHORIZATION_CONTEXT
         ).first
         logger.debug("Counted a total of $total entities for attribute $AUTH_PROP_SAP")
         entityService.searchEntities(
-            QueryParams(attrs = setOf(AUTH_PROP_SAP), offset = 0, limit = total),
+            QueryParams(
+                attrs = setOf(AUTH_PROP_SAP),
+                offset = 0,
+                limit = total,
+                context = NGSILD_AUTHORIZATION_CONTEXT
+            ),
             sub,
             NGSILD_AUTHORIZATION_CONTEXT
         ).second

diff --git a/...service/src/test/kotlin/com/egm/stellio/entity/authorization/AuthorizationServiceTests.kt b/...service/src/test/kotlin/com/egm/stellio/entity/authorization/AuthorizationServiceTests.kt
@@ -303,7 +303,7 @@ class AuthorizationServiceTests {
     @Test
     fun `get authorized entities should return a count of -1 if authentication is not enabled`() {
         val authorizedEntities = authorizationService.getAuthorizedEntities(
-            QueryParams(offset = 0, limit = 0),
+            QueryParams(offset = 0, limit = 0, context = NGSILD_CORE_CONTEXT),
             None,
             NGSILD_CORE_CONTEXT
         )