chore: update deployment guide for EDC 0 (#310)

sovity · Nov 17, 2023 · dda7763 · dda7763
1 parent daa9d33
commit dda7763
Show file tree

Hide file tree

Showing 10 changed files with 135 additions and 89 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -15,3 +15,7 @@ quote_type = single
 [*.md]
 max_line_length = off
 trim_trailing_whitespace = false
+
+[docker-compose.yaml]
+indent_size = 2
+
diff --git a/.env b/.env
@@ -1,4 +1,4 @@
 # Config for docker-compose.yaml
-BROKER_IMAGE=ghcr.io/sovity/broker-server-dev:1.2.0
-EDC_CE_IMAGE=ghcr.io/sovity/edc-dev:4.2.0
-EDC_UI_IMAGE=ghcr.io/sovity/edc-ui:0.0.1-milestone-8-sovity13
+BROKER_IMAGE=ghcr.io/sovity/broker-server-dev:latest
+EDC_IMAGE=ghcr.io/sovity/edc-dev:5.0.0
+EDC_UI_IMAGE=ghcr.io/sovity/edc-ui:2.0.0
diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md
@@ -43,4 +43,5 @@ Feel free to edit this release checklist in-progress depending on what tasks nee
     - [ ] Revert the versions in the [docker-compose's .env file](.env) back to latest for the Broker Server.
     - [ ] Update the [gradle.properties](https://github.com/sovity/edc-broker-server-extension/blob/main/gradle.properties) to contain the edc-extensions version `0.0.1-SNAPSHOT`.
     - [ ] Revisit the changed list of tasks and compare it with [.github/ISSUE_TEMPLATE/release.md](https://github.com/sovity/edc-broker-server-extension/blob/main/.github/ISSUE_TEMPLATE/release.md). Apply changes where it makes sense.
-    - [ ] Merge the `release-cleanup` PR.
+    - [ ] Merge the `release-cleanup` PR.
+- [ ] Close this issue.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,9 +24,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 #### Patch
 
+- Fixed some issues with DB Connections not released between tests.
+
 ### Deployment Migration Notes
 
-- All connectors need to be re-crawled for detailed asset metadata and participant IDs to work
+1. Connectors and Data Offers require an initial crawl before their metadata is filled again.
+2. Deployment Migration Notes for the Broker UI: https://github.com/sovity/edc-ui/releases/tag/v2.0.0
+3. The Protocol Endpoint changed to `https://[MY_EDC_FQDN]/backend/api/dsp`, ~~used to be `https://[MY_EDC_FQDN]/backend/api/v1/ids`~~.
+4. The Management Endpoint changed to `https://[MY_EDC_FQDN]/backend/api/management`, ~~used to be `https://[MY_EDC_FQDN]/backend/api/v1/management`~~.
+5. The Connector Endpoint changed to `https://[MY_EDC_FQDN]/backend/api/dsp`, ~~used to be `https://[MY_EDC_FQDN]/backend/api/v1/ids/data`~~.
 
 ## [v1.2.0] - 2023-10-30
 
@@ -93,7 +99,7 @@ Bugfix release for the asset properties issue. Also contains the connector delet
     curl --request DELETE \
         --url 'http://localhost:11002/backend/api/v1/management/wrapper/broker/connectors?adminApiKey=DefaultBrokerServerAdminApiKey' \
         --header 'Content-Type: application/json' \
-        --header 'X-Api-Key: ApiKeyDefaultValue' \
+        --header 'x-api-key: ApiKeyDefaultValue' \
         --data '["https://some-connector-to-delete/api/dsp", "https://some-other-connector-to-delete/api/dsp"]'
     ```
 
@@ -209,7 +215,7 @@ Bugfix / Feature Release for the Broker MvP with MS8: Connectors can now be adde
     curl --request PUT \
         --url 'http://localhost:11002/backend/api/v1/management/wrapper/broker/connectors?adminApiKey=DefaultBrokerServerAdminApiKey' \
         --header 'Content-Type: application/json' \
-        --header 'X-Api-Key: ApiKeyDefaultValue' \
+        --header 'x-api-key: ApiKeyDefaultValue' \
         --data '["https://some-new-connector/api/dsp", "https://some-other-new-connector/api/dsp"]'
     ```
    

diff --git a/README.md b/README.md
@@ -50,7 +50,11 @@ This IDS Broker is written on basis of the EDC and should be used in tandem with
 
 ## Development
 
-For development, access to the GitHub Maven Registry is required.
+### Local Development
+
+#### Local Backend Development
+
+For local backend development, access to the GitHub Maven Registry is required.
 
 To access the GitHub Maven Registry you need to provide the following properties, e.g. by providing
 a `~/.gradle/gradle.properties`.
@@ -60,11 +64,59 @@ gpr.user={your github username}
 gpr.key={your github pat with packages.read}
 ```
 
+Developing the Broker Backend tests are used to validate functionality:
+
+- There are Integration Tests using the Broker Server Java Client Library for testing API Endpoints of a running
+  backend.
+- There are Integration Tests using the Broker Server Java Client Library and sovity EDC Extensions to integration
+  test the Broker with a running EDC where communication works through the Data Space Protocol (DSP).
+- There are Unit Tests with Mockito for testing local complexity, e.g. mappers, data structures, utilities.
+
+<p align="right">(<a href="#readme-top">back to top</a>)</p>
+
+#### Local UI Development
+
+The Broker UI is a profile `broker` of the [EDC UI](https://github.com/sovity/edc-ui):
+
+The Broker UI depends on the NPM
+Package [@sovity/broker-server-client](https://www.npmjs.com/package/@sovity.de/broker-server-client) built on the main
+branch or on releases.
+
+Local Broker UI Development can start with the type-safe broker server fake backend once the Client Library version is
+bumped to contain the up-to-date API Models.
+
+<p align="right">(<a href="#readme-top">back to top</a>)</p>
+
+### Local E2E Development
+
+There is currently no support for Local E2E Development (a locally running backend build server and a locally running
+frontend build server).
+
+For debugging UI issues, however, the UI can be manually configured to use a live backend, e.g. one started via
+the [docker-compose.yaml](#local-demo).
+
+<p align="right">(<a href="#readme-top">back to top</a>)</p>
+
+### Local Demo
+
+There is a [docker-compose.yaml](docker-compose.yaml) that starts a broker and a connector.
+
+At release time it is pinned down to the release versions.
+
+Mid-development it might be un-pinned back to latest versions.
+
+|                     | Broker                                                           | Conncetor                                                                    |
+|---------------------|------------------------------------------------------------------|:-----------------------------------------------------------------------------|
+| Homepage            | http://localhost:11000                                           | http://localhost:22000                                                       |
+| Management Endpoint | http://localhost:11002/api/management                            | http://localhost:22002/api/management                                        |
+| Management API Key  | `ApiKeyDefaultValue`                                             | `ApiKeyDefaultValue`                                                         |
+| Connector Endpoint  | http://broker:11003/api/dsp <br> Requires Docker Compose Network | http://connector:22003/api/dsp          <br> Requires Docker Compose Network |
+
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
 ## Releasing
 
-Create an issue using the [release template](.github/ISSUE_TEMPLATE/release.md) and follow the instructions.
+[Create a Release Issue](https://github.com/sovity/edc-broker-server-extension/issues/new?assignees=&labels=task%2Frelease%2Cscope%2Fmds&projects=&template=release.md&title=Release+x.x.x) and follow the instructions.
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
@@ -89,13 +141,13 @@ or if it's broken.
 
 - The broker is meant to be served via TLS/HTTPS.
 - The broker is meant to be deployed with a reverse proxy merging the following ports:
-    - The UI's `80` port.
+    - The UI's `8080` port.
     - The Backend's `11002` port.
     - The Backend's `11003` port.
 - The mapping should look like this:
-    - `/backend/api/v1/ids` -> `broker-backend:11003/backend/api/v1/ids`
-    - `/backend/api/v1/management` -> `broker-backend:11002/backend/api/v1/management`
-    - All other requests should be mapped to `broker-ui:80`
+    - `https://[MY_EDC_FQDN]/backend/api/dsp` -> `broker-backend:11003/backend/api/dsp`
+    - `https://[MY_EDC_FQDN]/backend/api/management` -> `broker-backend:11002/backend/api/management`
+    - All other requests -> `broker-ui:8080`
 
 #### Backend Configuration
 
@@ -146,10 +198,10 @@ in [connector/.env](connector/.env).
 EDC_UI_ACTIVE_PROFILE: broker
 
 # Required: Management API URL
-EDC_UI_DATA_MANAGEMENT_API_URL: https://my-broker.com/backend/api/v1/management
+EDC_UI_MANAGEMENT_API_URL: https://my-broker.com/backend/api/management
 
 # Required: Management API Key
-EDC_API_AUTH_KEY: "ApiKeyDefaultValue"
+EDC_UI_MANAGEMENT_API_KEY: "ApiKeyDefaultValue"
 ```
 
 #### Adding Connectors at runtime
@@ -159,9 +211,9 @@ Connectors can be dynamically added at runtime by using the following endpoint:
 ```shell script
 # Response should be 204 No Content
 curl --request PUT \
-  --url 'http://localhost:11002/backend/api/v1/management/wrapper/broker/connectors?adminApiKey=DefaultBrokerServerAdminApiKey' \
+  --url 'http://localhost:11002/backend/api/management/wrapper/broker/connectors?adminApiKey=DefaultBrokerServerAdminApiKey' \
   --header 'Content-Type: application/json' \
-  --header 'X-Api-Key: ApiKeyDefaultValue' \
+  --header 'x-api-key: ApiKeyDefaultValue' \
   --data '["https://some-new-connector/api/dsp", "https://some-other-new-connector/api/dsp"]'
 ```
 
@@ -172,9 +224,9 @@ Connectors can be dynamically removed at runtime by using the following endpoint
 ```shell script
 # Response should be 204 No Content
 curl --request DELETE \
-  --url 'http://localhost:11002/backend/api/v1/management/wrapper/broker/connectors?adminApiKey=DefaultBrokerServerAdminApiKey' \
+  --url 'http://localhost:11002/backend/api/management/wrapper/broker/connectors?adminApiKey=DefaultBrokerServerAdminApiKey' \
   --header 'Content-Type: application/json' \
-  --header 'X-Api-Key: ApiKeyDefaultValue' \
+  --header 'x-api-key: ApiKeyDefaultValue' \
   --data '["https://some-connector-to-be-removed/api/dsp", "https://some-other-connector-to-be-removed/api/dsp"]'
 ```
 

diff --git a/connector/.env b/connector/.env
@@ -79,59 +79,20 @@ WEB_HTTP_MANAGEMENT_PORT=11002
 WEB_HTTP_PROTOCOL_PORT=11003
 WEB_HTTP_CONTROL_PORT=11004
 WEB_HTTP_PATH=${MY_EDC_BASE_PATH}/api
-WEB_HTTP_MANAGEMENT_PATH=${MY_EDC_BASE_PATH}/api/v1/management
-WEB_HTTP_PROTOCOL_PATH=${MY_EDC_BASE_PATH}/api/v1/ids
-WEB_HTTP_CONTROL_PATH=${MY_EDC_BASE_PATH}/api/v1/control
+WEB_HTTP_MANAGEMENT_PATH=${MY_EDC_BASE_PATH}/api/management
+WEB_HTTP_PROTOCOL_PATH=${MY_EDC_BASE_PATH}/api/dsp
+WEB_HTTP_CONTROL_PATH=${MY_EDC_BASE_PATH}/api/control
 
 EDC_CONNECTOR_NAME=$MY_EDC_NAME_KEBAB_CASE
 EDC_HOSTNAME=${MY_EDC_FQDN}
-
-# Deprecated IDS Settings
-EDC_IDS_ID=urn:connector:$MY_EDC_NAME_KEBAB_CASE
-EDC_IDS_TITLE=This will be unavailable starting Core EDC 0.1.0
-EDC_IDS_DESCRIPTION=This will be unavailable starting Core EDC 0.1.0
-EDC_IDS_CURATOR=http://this-will-be-unavailable-starting-core-edc-0-1-0
-EDC_IDS_MAINTAINER=http://this-will-be-unavailable-starting-core-edc-0-1-0
-MY_EDC_IDS_BASE_URL=${MY_EDC_PROTOCOL}${MY_EDC_FQDN}
-IDS_WEBHOOK_ADDRESS=${MY_EDC_IDS_BASE_URL}
-EDC_IDS_ENDPOINT=${MY_EDC_IDS_BASE_URL}${WEB_HTTP_PROTOCOL_PATH}
+EDC_DSP_CALLBACK_ADDRESS=${MY_EDC_PROTOCOL}${MY_EDC_FQDN}${WEB_HTTP_PROTOCOL_PATH}
 
 # Flyway Extension: Defaults
 EDC_DATASOURCE_DEFAULT_NAME=default
 EDC_DATASOURCE_DEFAULT_URL=$MY_EDC_JDBC_URL
 EDC_DATASOURCE_DEFAULT_USER=$MY_EDC_JDBC_USER
 EDC_DATASOURCE_DEFAULT_PASSWORD=$MY_EDC_JDBC_PASSWORD
 
-EDC_DATASOURCE_ASSET_NAME=asset
-EDC_DATASOURCE_ASSET_URL=$MY_EDC_JDBC_URL
-EDC_DATASOURCE_ASSET_USER=$MY_EDC_JDBC_USER
-EDC_DATASOURCE_ASSET_PASSWORD=$MY_EDC_JDBC_PASSWORD
-
-EDC_DATASOURCE_CONTRACTDEFINITION_NAME=contractdefinition
-EDC_DATASOURCE_CONTRACTDEFINITION_URL=$MY_EDC_JDBC_URL
-EDC_DATASOURCE_CONTRACTDEFINITION_USER=$MY_EDC_JDBC_USER
-EDC_DATASOURCE_CONTRACTDEFINITION_PASSWORD=$MY_EDC_JDBC_PASSWORD
-
-EDC_DATASOURCE_CONTRACTNEGOTIATION_NAME=contractnegotiation
-EDC_DATASOURCE_CONTRACTNEGOTIATION_URL=$MY_EDC_JDBC_URL
-EDC_DATASOURCE_CONTRACTNEGOTIATION_USER=$MY_EDC_JDBC_USER
-EDC_DATASOURCE_CONTRACTNEGOTIATION_PASSWORD=$MY_EDC_JDBC_PASSWORD
-
-EDC_DATASOURCE_POLICY_NAME=policy
-EDC_DATASOURCE_POLICY_URL=$MY_EDC_JDBC_URL
-EDC_DATASOURCE_POLICY_USER=$MY_EDC_JDBC_USER
-EDC_DATASOURCE_POLICY_PASSWORD=$MY_EDC_JDBC_PASSWORD
-
-EDC_DATASOURCE_TRANSFERPROCESS_NAME=transferprocess
-EDC_DATASOURCE_TRANSFERPROCESS_URL=$MY_EDC_JDBC_URL
-EDC_DATASOURCE_TRANSFERPROCESS_USER=$MY_EDC_JDBC_USER
-EDC_DATASOURCE_TRANSFERPROCESS_PASSWORD=$MY_EDC_JDBC_PASSWORD
-
-EDC_DATASOURCE_DATAPLANEINSTANCE_NAME=dataplaneinstance
-EDC_DATASOURCE_DATAPLANEINSTANCE_URL=$MY_EDC_JDBC_URL
-EDC_DATASOURCE_DATAPLANEINSTANCE_USER=$MY_EDC_JDBC_USER
-EDC_DATASOURCE_DATAPLANEINSTANCE_PASSWORD=$MY_EDC_JDBC_PASSWORD
-
 # Oauth default configurations
 EDC_OAUTH_PROVIDER_AUDIENCE=idsc:IDS_CONNECTORS_ALL
 

diff --git a/connector/Dockerfile b/connector/Dockerfile
@@ -34,7 +34,7 @@ COPY --from=build /home/gradle/project/connector/build/libs/app.jar /app
 COPY ./connector/src/main/resources/logging.properties /app
 
 # health status is determined by the availability of the /health endpoint
-HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "X-Api-Key: $EDC_API_AUTH_KEY" --fail http://localhost:11002/backend/api/v1/management/check/health
+HEALTHCHECK --interval=5s --timeout=5s --retries=10 CMD curl -H "x-api-key: $EDC_API_AUTH_KEY" --fail http://localhost:11002/backend/api/v1/management/check/health
 
 # Use "exec" for graceful termination (SIGINT) to reach JVM.
 # ARG can not be used in ENTRYPOINT so storing values in ENV variables

diff --git a/connector/build.gradle.kts b/connector/build.gradle.kts
@@ -10,6 +10,7 @@ val edcGroup: String by project
 dependencies {
     // Control-Plane
     implementation("${edcGroup}:control-plane-core:${edcVersion}")
+    implementation("${edcGroup}:data-plane-selector-core:${edcVersion}")
     implementation("${edcGroup}:api-observability:${edcVersion}")
     implementation("${edcGroup}:configuration-filesystem:${edcVersion}")
     implementation("${edcGroup}:control-plane-aggregate-services:${edcVersion}")

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -3,33 +3,37 @@ services:
   broker-ui:
     image: ${EDC_UI_IMAGE}
     ports:
-      - '11000:80'
+      - '11000:8080'
     environment:
-      - EDC_UI_ACTIVE_PROFILE=broker
-      - EDC_UI_DATA_MANAGEMENT_API_URL=http://localhost:11002/backend/api/v1/management
-      - EDC_UI_DATA_MANAGEMENT_API_KEY=ApiKeyDefaultValue
+      EDC_UI_ACTIVE_PROFILE: broker
+      EDC_UI_MANAGEMENT_API_URL: http://localhost:11002/backend/api/management
+      EDC_UI_MANAGEMENT_API_KEY: ApiKeyDefaultValue
+      NGINX_ACCESS_LOG: off
   broker:
     image: ${BROKER_IMAGE}
     depends_on:
       - broker-postgresql
       - connector
     environment:
-      # Broker Configuration
+      EDC_BROKER_SERVER_KNOWN_CONNECTORS: "http://connector:11003/api/dsp"
+      EDC_BROKER_SERVER_KNOWN_DATASPACE_CONNECTORS: "Mobilithek=https://some-other-connector/api/dsp"
+
+      # Hide offline data offers after 1 minute in dev
+      EDC_BROKER_SERVER_HIDE_OFFLINE_DATA_OFFERS_AFTER: "PT1M"
+
       MY_EDC_FQDN: "broker"
+      EDC_API_AUTH_KEY: ApiKeyDefaultValue
+
       MY_EDC_JDBC_URL: jdbc:postgresql://broker-postgresql:5432/edc
       MY_EDC_JDBC_USER: edc
       MY_EDC_JDBC_PASSWORD: edc
-      EDC_BROKER_SERVER_KNOWN_CONNECTORS: "http://connector:11003/api/dsp"
-      EDC_BROKER_SERVER_KNOWN_DATASPACE_CONNECTORS: "Mobilithek=https://some-other-connector/api/dsp"
 
-      # Local Dev / Docker-Compose Config
-      MY_EDC_PROTOCOL: "http://" # We don't have TLS in the docker container
-      MY_EDC_IDS_BASE_URL: "http://broker:11003" # Add the port, because we have no reverse proxy erasing the ports here
+      # docker compose local dev environment overrides (don't use with non-dev images)
+      MY_EDC_PROTOCOL: "http://"
+      EDC_DSP_CALLBACK_ADDRESS: http://broker:11003/backend/api/dsp
       EDC_WEB_REST_CORS_ENABLED: 'true'
       EDC_WEB_REST_CORS_HEADERS: 'origin,content-type,accept,authorization,x-api-key'
       EDC_WEB_REST_CORS_ORIGINS: '*'
-      EDC_BROKER_SERVER_HIDE_OFFLINE_DATA_OFFERS_AFTER: "PT1M" # Hide offline data offers after 1 minute in dev
-      EDC_API_AUTH_KEY: "ApiKeyDefaultValue" # Management API Key (Access to UI should be secured by other means, as this key is sent to the UI)
     ports:
       - '11001:11001'
       - '11002:11002'
@@ -44,36 +48,42 @@ services:
       POSTGRESQL_PASSWORD: edc
       POSTGRESQL_DATABASE: edc
     ports:
-        - '54321:5432'
+      - '54321:5432'
     volumes:
       - 'broker-postgresql:/bitnami/postgresql'
   connector-ui:
     image: ${EDC_UI_IMAGE}
     ports:
-      - '22000:80'
+      - '22000:8080'
     environment:
-      - EDC_UI_ACTIVE_PROFILE=mds-open-source
-      - EDC_UI_CONFIG_URL=edc-ui-config
-      - EDC_UI_DATA_MANAGEMENT_API_URL=http://localhost:22002/api/v1/management
-      - EDC_UI_DATA_MANAGEMENT_API_KEY=ApiKeyDefaultValue
+      EDC_UI_ACTIVE_PROFILE: mds-open-source
+      EDC_UI_CONFIG_URL: edc-ui-config
+      EDC_UI_MANAGEMENT_API_URL: http://localhost:22002/api/v1/management
+      EDC_UI_MANAGEMENT_API_KEY: ApiKeyDefaultValue
+      NGINX_ACCESS_LOG: off
   connector:
-    image: ${EDC_CE_IMAGE}
+    image: ${EDC_IMAGE}
+    depends_on:
+      - connector-postgresql
     environment:
-      MY_EDC_NAME_KEBAB_CASE: "example-connector"
+      MY_EDC_NAME_KEBAB_CASE: "my-connector"
       MY_EDC_TITLE: "EDC Connector"
-      MY_EDC_DESCRIPTION: "MDS Community Edition EDC Connector"
+      MY_EDC_DESCRIPTION: "sovity Community Edition EDC Connector"
       MY_EDC_CURATOR_URL: "https://example.com"
       MY_EDC_CURATOR_NAME: "Example GmbH"
       MY_EDC_MAINTAINER_URL: "https://sovity.de"
       MY_EDC_MAINTAINER_NAME: "sovity GmbH"
 
-      # Data Management API Key
+      MY_EDC_FQDN: "connector"
       EDC_API_AUTH_KEY: ApiKeyDefaultValue
 
-      # Local Dev / Docker-Compose Config
+      MY_EDC_JDBC_URL: jdbc:postgresql://connector-postgresql:5432/edc
+      MY_EDC_JDBC_USER: edc
+      MY_EDC_JDBC_PASSWORD: edc
+
+      # docker compose local dev environment overrides (don't use with non-dev images)
       MY_EDC_PROTOCOL: "http://"
-      MY_EDC_FQDN: "connector"
-      MY_EDC_IDS_BASE_URL: "http://connector:11003"
+      EDC_DSP_CALLBACK_ADDRESS: http://connector:11003/api/dsp
       EDC_WEB_REST_CORS_ENABLED: 'true'
       EDC_WEB_REST_CORS_HEADERS: 'origin,content-type,accept,authorization,x-api-key'
       EDC_WEB_REST_CORS_ORIGINS: '*'
@@ -83,6 +93,17 @@ services:
       - '22003:11003'
       - '22004:11004'
       - '22005:5005'
+  connector-postgresql:
+    image: docker.io/bitnami/postgresql:11
+    restart: always
+    environment:
+      POSTGRESQL_USERNAME: edc
+      POSTGRESQL_PASSWORD: edc
+      POSTGRESQL_DATABASE: edc
+    ports:
+      - '54322:5432'
+    volumes:
+      - 'connector-postgresql:/bitnami/postgresql'
 volumes:
   broker-postgresql:
     driver: local