Update schema tools to fix mock data (#930)

* Update schema tools to fix mock data When running docker compose with mock data the command failed due to an outdated schema being provided. We now use the active Amsterdam schema to generate mock data. The provided schema container is now also obsolute and thus removed. * Use local schemas in openapi spec test Starting up the container with the full Amsterdam Schema takes to long, so we use the schemas container for this test using a specific docker compose override. The schemas have been updated to a recent version.
Amsterdam · Jan 22, 2025 · 6289093 · 6289093
1 parent c74aa0a
commit 6289093
Show file tree

Hide file tree

Showing 59 changed files with 4,616 additions and 1,990 deletions.
diff --git a/.github/workflows/test-openapi-spec.yaml b/.github/workflows/test-openapi-spec.yaml
@@ -19,7 +19,10 @@ jobs:
       uses: actions/checkout@v1
 
     - name: Start containers
-      run: docker compose -f "docker-compose.yml" up -d --build
+      run: docker compose -f "docker-compose.yml" -f "docker-compose.test-openapi-spec.yml" up -d --build
+      env:
+        SCHEMA_URL: "http://schemas/datasets/"
+
 
     - name: Setup tests
       run: |
@@ -28,6 +31,8 @@ jobs:
         echo 'extends: ["spectral:oas", "spectral:asyncapi"]' > ./spectral/.spectral.yaml
         # Give dso-api some time to start
         sleep 10
+        docker compose logs web
+
         # Grab openapi yaml for bag dataset
         docker compose exec web curl 'http://web:8000/v1/bag/?format=json'  > ./spectral/openapi.yaml
 
@@ -36,4 +41,4 @@ jobs:
 
     - name: Stop containers
       if: always()
-      run: docker compose -f "docker-compose.yml" down
+      run: docker compose -f "docker-compose.yml" -f "docker-compose.test-openapi-spec.yml" down
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -30,7 +30,7 @@ repos:
         args:
           - --target-version=4.2
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.2
     hooks:
       - id: ruff
         args:

diff --git a/README.md b/README.md
@@ -35,8 +35,12 @@ The following environment variables are useful for configuring a local developme
 * `SCHEMA_URL`: URL of the dataset schemas. To use the production schemas set this to:
     "https://schemas.data.amsterdam.nl/datasets/"
     Default is the `schemas` container in the compose file.
+* `DATASETS_INCLUDE`: A comma separated lists of datasets to expose using the API or to generate mock data for.
+    Default: `None` (expose all loaded datasets).
+* `DATASETS_INCLUDE`: A comma separated lists of datasets to expose using the API or to generate mock data for.
+    Default: `None` (expose all loaded datasets).
 
-To connect to an authentication provider, setup the following environment variables:
+To connect to an authentication provider, set up the following environment variables:
 * `OAUTH_CLIENT_ID`: The client id of the application
 * `OAUTH_JWKS_URL`: The JWKS URL of the authentication provider.
 * `OAUTH_URL`:  The auth URL of the authentication provider.
@@ -46,8 +50,9 @@ A local setup using the production dataset schema's with mock data:
 ```
     export INITIALIZE_DB=true
     export MOCK_DATA=true
+    export DATASETS_LIST=bag,gebieden,brk_2,sport,precariobelasting
     export SCHEMA_URL="https://schemas.data.amsterdam.nl/datasets/"
-    compose up
+    docker compose up
 ```
 
 ## Without Docker Compose

diff --git a/dev-docs/source/environment.rst b/dev-docs/source/environment.rst
@@ -25,13 +25,13 @@ By default, all datasets are hosted by the application.
 When a custom instance is deployed for a particular dataset (e.g. the BRP),
 this instance can host only a subset using::
 
-    DATASETS_LISTS = ...
+    DATASETS_LIST = ...
     DATASETS_EXCLUDE = ...
 
 To expose only a subset of the datasets, use ``DATASETS_LIST``.
 To expose all datasets with some exceptions, use ``DATASETS_EXCLUDE``.
 
-Both entries accept a comma separated list, e.g. ``DATASETS_LISTS=bommen,gebieden,meldingen``.
+Both entries accept a comma separated list, e.g. ``DATASETS_LIST=bommen,gebieden,meldingen``.
 
 .. warning::
 

diff --git a/docker-compose.test-openapi-spec.yml b/docker-compose.test-openapi-spec.yml
@@ -0,0 +1,17 @@
+services:
+  web:
+    links:
+      - database
+      - schemas
+    environment:
+      SCHEMA_URL: "${SCHEMA_URL:-http://schemas/datasets/}"
+      DATASETS_LIST: "bag,brk_2,gebieden,sportparken,precariobelasting"
+
+  schemas:
+    image: nginx
+    ports:
+      - "8080:80"
+    volumes:
+      - "./schemas/data/datasets:/usr/share/nginx/html/datasets"
+      - "./schemas/data/publishers:/usr/share/nginx/html/publishers"
+      - "./schemas/conf/default.conf:/etc/nginx/conf.d/default.conf"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -27,7 +27,6 @@ services:
       - "8090:8000"
     links:
       - database
-      - schemas
     depends_on:
       database:
         condition: service_healthy
@@ -56,7 +55,9 @@ services:
       APIKEY_ENABLED: "${APIKEY_ENABLED}"
       INITIALIZE_DB: "${INITIALIZE_DB:-true}"
       MOCK_DATA: "${MOCK_DATA:-false}"
-      SCHEMA_URL: "http://schemas/datasets/" # "https://schemas.data.amsterdam.nl/datasets/"
+      SCHEMA_URL: "${SCHEMA_URL:-https://schemas.data.amsterdam.nl/datasets/}"
+      DATASETS_LIST: "${DATASETS_LIST}"
+      DATASETS_EXCLUDE: "${DATASETS_EXCLUDE}"
     volumes:
       - ./src:/app
       - ./docs/build/html:/docs-static
@@ -71,12 +72,3 @@ services:
       - ALLOW_EMPTY_PASSWORD=yes
     ports:
       - '6379:6379'
-
-  schemas:
-    image: nginx
-    ports:
-      - "8080:80"
-    volumes:
-      - "./schemas/data/datasets:/usr/share/nginx/html/datasets"
-      - "./schemas/data/publishers:/usr/share/nginx/html/publishers"
-      - "./schemas/conf/default.conf:/etc/nginx/conf.d/default.conf"
diff --git a/schemas/data/datasets/bag/brondocumenten/v1.0.1 b/schemas/data/datasets/bag/brondocumenten/v1.0.1
@@ -0,0 +1,68 @@
+{
+	"id": "brondocumenten",
+	"type": "table",
+	"version": "1.0.1",
+	"schema": {
+		"$schema": "http://json-schema.org/draft-07/schema#",
+		"type": "object",
+		"additionalProperties": false,
+		"identifier": "documentnummer",
+		"required": [
+			"schema",
+			"documentnummer"
+		],
+		"display": "documentnummer",
+		"properties": {
+			"schema": {
+				"$ref": "https://schemas.data.amsterdam.nl/[email protected]#/definitions/schema"
+			},
+			"documentnummer": {
+				"type": "string",
+				"title": "Brondocument nummer",
+				"description": "Identificerende nummer van het document"
+			},
+			"bronleverancierCode": {
+				"type": "string",
+				"provenance": "$.bronleverancier.code",
+				"title": "Bron leverancier code",
+				"description": "Verstrekker van brondocumenten en/of gegevens voortkomend uit het uitoefenen of voorbereiden van een gemeentelijke bevoegdheid, die nodig zijn voor een registratie aan de bronhouder, conform vastgestelde aanleverspecificaties. code"
+			},
+			"bronleverancierOmschrijving": {
+				"type": "string",
+				"provenance": "$.bronleverancier.omschrijving",
+				"title": "Bron leverancier omschrijving",
+				"description": "Verstrekker van brondocumenten en/of gegevens voortkomend uit het uitoefenen of voorbereiden van een gemeentelijke bevoegdheid, die nodig zijn voor een registratie aan de bronhouder, conform vastgestelde aanleverspecificaties. omschrijving"
+			},
+			"typeDossierCode": {
+				"type": "string",
+				"provenance": "$.typeDossier.code",
+				"title": "Dossier type code",
+				"description": "Het type dossier. code"
+			},
+			"typeDossierOmschrijving": {
+				"type": "string",
+				"provenance": "$.typeDossier.omschrijving",
+				"title": "Dossier type omschrijving",
+				"description": "Het type dossier. omschrijving"
+			},
+			"typeBrondocumentCode": {
+				"type": "string",
+				"provenance": "$.typeBrondocument.code",
+				"title": "Brondocument type code",
+				"description": "Het type brondocument. code"
+			},
+			"typeBrondocumentOmschrijving": {
+				"type": "string",
+				"provenance": "$.typeBrondocument.omschrijving",
+				"title": "Brondocument type omschrijving",
+				"description": "Het type brondocument. omschrijving"
+			},
+			"registratiedatum": {
+				"type": "string",
+				"title": "Registratie datum",
+				"format": "date-time",
+				"description": "De datum waarop het brondocument is opgeslagen in het register"
+			}
+		}
+	}
+}