From 13052ef7158b55249a21aba386cb6cbbe599d14a Mon Sep 17 00:00:00 2001 From: sambles Date: Wed, 7 Feb 2024 11:42:07 +0000 Subject: [PATCH] Update/platform v2 (#44) * Update compose for version 2.3.0 * Fix port clash * Fix model mount * Set UI version to v2 * Fix images and update swagger links * Fix GH? * foo * Fix workflow later --- .github/workflows/install-test.yaml | 64 ++++----- README.md | 32 +++-- install.sh | 8 +- oasis-platform.yml | 206 ++++++++++++++++++---------- oasis-ui-proxy.yml | 2 +- oasis-ui-standalone.yml | 2 +- portainer.yaml | 2 +- 7 files changed, 192 insertions(+), 124 deletions(-) diff --git a/.github/workflows/install-test.yaml b/.github/workflows/install-test.yaml index e32e444..bea8a57 100644 --- a/.github/workflows/install-test.yaml +++ b/.github/workflows/install-test.yaml @@ -1,33 +1,35 @@ name: Evaluation Installation Test -on: - push: - branches: - - main - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - install_test: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: run installation - run: ./install.sh - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: install test deps - run: | - python -m pip install --upgrade pip - pip install -r OasisPiWind/tests/requirements.in - - - name: run PiWind test - run: pytest OasisPiWind/tests/test_piwind_integration.py -v --use-running-containers --docker-compose=oasis-platform.yml -k all_outputs +#on: +# push: +# branches: +# - main +# pull_request: +# branches: +# - main +# workflow_dispatch: +# +#jobs: +# install_test: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout +# uses: actions/checkout@v3 +# +# - name: run installation +# run: ./install.sh +# +# - name: Setup Python +# uses: actions/setup-python@v4 +# with: +# python-version: '3.10' +# +# - name: install test deps +# run: | +# python -m pip install --upgrade pip +# pip install -r OasisPiWind/tests/requirements.in +# +# - name: run PiWind test +# run: pytest OasisPiWind/tests/test_piwind_integration.py -v --use-running-containers --docker-compose=oasis-platform.yml -k all_outputs +# env: +# WORKER_API_VER: 'v1' diff --git a/README.md b/README.md index 8ac22f3..663bc4c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The platform can be run via docker containers on a single machine or, if require Docker support is the main requirement for running the platform. A Linux based installation is the main focus of this example deployment. -Running the install script from this repository automates install process of the OasisPlatform API v1, User Interface and example PiWind model. +Running the install script from this repository automates install process of the OasisPlatform API v2, User Interface and example PiWind model. ## Install Prerequisites * Host operating system with docker support, see [Docker compatibility matrix](https://success.docker.com/article/compatibility-matrix). @@ -43,7 +43,7 @@ sudo ./install.sh #### Oasis Docker Components -![Oasis docker images](https://github.com/OasisLMF/OasisEvaluation/raw/master/.img/oasis_containers.png) +![Oasis docker images](https://github.com/OasisLMF/OasisEvaluation/raw/main/.img/oasis_containers.png) |Component |Description |Technology | | ---------------------:| ------------------:| ----------:| @@ -66,27 +66,33 @@ Pass: password ``` ### [OasisUI Interface](http://localhost:8080/) - *localhost:8080/* -![alt text](https://github.com/OasisLMF/OasisEvaluation/raw/master/.img/oasisui.png) +![alt text](https://github.com/OasisLMF/OasisEvaluation/raw/main/.img/oasisui.png) + + +### [API Swagger UI - All](http://localhost:8000/) - *localhost:8000* +### [API Swagger UI - V1](http://localhost:8000/v1/) - *localhost:8000/v1/* +### [API Swagger UI - V2](http://localhost:8000/v2/) - *localhost:8000/v2/* +![alt text](https://github.com/OasisLMF/OasisEvaluation/raw/main/.img/api_swagger.png) + + -### [API Swagger UI](http://localhost:8000/) - *localhost:8000* -![alt text](https://github.com/OasisLMF/OasisEvaluation/raw/master/.img/api_swagger.png) ### [API Admin Panel](http://localhost:8000/admin) - *localhost:8000/admin* -![alt text](https://github.com/OasisLMF/OasisEvaluation/raw/master/.img/admin_panel.png) +![alt text](https://github.com/OasisLMF/OasisEvaluation/raw/main/.img/admin_panel.png) ### [Portainer Docker Panel](http://localhost:9000) - *localhost:9000* -![alt text](https://github.com/OasisLMF/OasisAtScaleEvaluation/raw/master/.img/portainer.png) +![alt text](https://github.com/OasisLMF/OasisAtScaleEvaluation/raw/main/.img/portainer.png) ## Exposure inputs The Oasis platform supports the [Open Exposure Data (OED)](https://github.com/Simplitium/OED) standard for importing exposure. Example files are available for the PiWind model: -* [SourceLocOEDPiWind10.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/master/tests/inputs/SourceLocOEDPiWind10.csv) --- Locations Data 10 rows -* [SourceLocOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/master/tests/inputs/SourceLocOEDPiWind.csv) --- Locations Data -* [SourceAccOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/master/tests/inputs/SourceAccOEDPiWind.csv) --- Accounts Data -* [SourceReinsInfoOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/master/tests/inputs/SourceReinsInfoOEDPiWind.csv) --- Reinsurance Info -* [SourceReinsScopeOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/master/tests/inputs/SourceReinsScopeOEDPiWind.csv) --- Reinsurance Scope +* [SourceLocOEDPiWind10.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/main/tests/inputs/SourceLocOEDPiWind10.csv) --- Locations Data 10 rows +* [SourceLocOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/main/tests/inputs/SourceLocOEDPiWind.csv) --- Locations Data +* [SourceAccOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/main/tests/inputs/SourceAccOEDPiWind.csv) --- Accounts Data +* [SourceReinsInfoOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/main/tests/inputs/SourceReinsInfoOEDPiWind.csv) --- Reinsurance Info +* [SourceReinsScopeOEDPiWind.csv](https://raw.githubusercontent.com/OasisLMF/OasisPiWind/main/tests/inputs/SourceReinsScopeOEDPiWind.csv) --- Reinsurance Scope ## Troubleshooting Feedback and error reports are invaluable for improving the stability and performance of the Oasis Platform, If you encounter an issue please consider [submitting an issue here](https://github.com/OasisLMF/OasisPlatform/issues) @@ -169,6 +175,6 @@ This evaluation also includes a Jupyter notebook that accesses the API using the The following sequence diagram illustrates how the general steps in the analysis, encapulated by the API client functions, map to specific API calls. -![Oasis API sequence](https://github.com/OasisLMF/OasisEvaluation/raw/master/.img/oasis_api_sequence.png) +![Oasis API sequence](https://github.com/OasisLMF/OasisEvaluation/raw/main/.img/oasis_api_sequence.png) diff --git a/install.sh b/install.sh index d5644ee..be8ef07 100755 --- a/install.sh +++ b/install.sh @@ -3,11 +3,11 @@ set -e SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export VERS_MDK=1.28.4 -export VERS_API=1.28.4 -export VERS_WORKER=1.28.4 +export VERS_MDK=2.3.0 +export VERS_API=2.3.0 +export VERS_WORKER=2.3.0 export VERS_UI=1.11.7 -export VERS_PIWIND='stable/1.28.x' +export VERS_PIWIND='stable/2.3.x' export SERVER_IMG=coreoasis/api_server export WORKER_IMG=coreoasis/model_worker diff --git a/oasis-platform.yml b/oasis-platform.yml index 6a6fc9b..8f70aef 100755 --- a/oasis-platform.yml +++ b/oasis-platform.yml @@ -3,89 +3,154 @@ volumes: server-db-OasisData: celery-db-OasisData: filestore-OasisData: +x-shared-env: &shared-env + OASIS_DEBUG: 1 + OASIS_URL_SUB_PATH: 0 + OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" + OASIS_SERVER_DB_HOST: server-db + OASIS_SERVER_DB_PASS: oasis + OASIS_SERVER_DB_USER: oasis + OASIS_SERVER_DB_NAME: oasis + OASIS_SERVER_DB_PORT: 5432 + OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer + OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2 + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 + OASIS_CELERY_DB_HOST: celery-db + OASIS_CELERY_DB_PASS: password + OASIS_CELERY_DB_USER: celery + OASIS_CELERY_DB_NAME: celery + OASIS_CELERY_DB_PORT: 5432 + OASIS_INPUT_GENERATION_CONTROLLER_QUEUE: task-controller + OASIS_LOSSES_GENERATION_CONTROLLER_QUEUE: task-controller + +x-oasis-env-v1: &oasis-env-v1 + OASIS_DEBUG: ${DEBUG:-0} + OASIS_RABBIT_HOST: broker + OASIS_RABBIT_PORT: 5672 + OASIS_RABBIT_USER: rabbit + OASIS_RABBIT_PASS: rabbit + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 + OASIS_CELERY_DB_HOST: celery-db + OASIS_CELERY_DB_PASS: password + OASIS_CELERY_DB_USER: celery + OASIS_CELERY_DB_NAME: celery + OASIS_CELERY_DB_PORT: 5432 + +x-volumes: &shared-volumes + - filestore-OasisData:/shared-fs:rw services: server: restart: always image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} + command: ["./wsgi/run-wsgi.sh"] ports: - 8000:8000 + - 51970:51970 links: - server-db - celery-db - - rabbit + - broker environment: - - OASIS_ADMIN_USER=admin - - OASIS_ADMIN_PASS=password - - OASIS_DEBUG=1 - - OASIS_RABBIT_HOST=rabbit - - OASIS_RABBIT_PORT=5672 - - OASIS_RABBIT_USER=rabbit - - OASIS_RABBIT_PASS=rabbit - - OASIS_SERVER_DB_HOST=server-db - - OASIS_SERVER_DB_PASS=oasis - - OASIS_SERVER_DB_USER=oasis - - OASIS_SERVER_DB_NAME=oasis - - OASIS_SERVER_DB_PORT=5432 - - OASIS_SERVER_DB_ENGINE=django.db.backends.postgresql_psycopg2 - - OASIS_CELERY_DB_ENGINE=db+postgresql+psycopg2 - - OASIS_CELERY_DB_HOST=celery-db - - OASIS_CELERY_DB_PASS=password - - OASIS_CELERY_DB_USER=celery - - OASIS_CELERY_DB_NAME=celery - - OASIS_CELERY_DB_PORT=5432 - - STARTUP_RUN_MIGRATIONS=true + <<: *shared-env + STARTUP_RUN_MIGRATIONS: "true" + OASIS_ADMIN_USER: admin + OASIS_ADMIN_PASS: password volumes: - filestore-OasisData:/shared-fs:rw - worker-monitor: + server_websocket: restart: always image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} - command: [wait-for-server, 'server:8000', celery, -A, src.server.oasisapi, worker, --loglevel=INFO] + command: ["./asgi/run-asgi.sh"] links: - server-db - celery-db - - rabbit + - broker + ports: + - 8001:8001 + environment: + <<: *shared-env + volumes: + - filestore-OasisData:/shared-fs:rw + v1-worker-monitor: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} + command: [celery, -A, 'src.server.oasisapi.celery_app_v1', worker, --loglevel=INFO,] + links: + - server-db + - celery-db + - broker + environment: + <<: *shared-env + volumes: + - filestore-OasisData:/shared-fs:rw + v2-worker-monitor: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} + command: [celery, -A, 'src.server.oasisapi.celery_app_v2', worker, --loglevel=INFO, -Q, celery-v2] + links: + - server-db + - celery-db + - broker environment: - - OASIS_DEBUG=1 - - OASIS_RABBIT_HOST=rabbit - - OASIS_RABBIT_PORT=5672 - - OASIS_RABBIT_USER=rabbit - - OASIS_RABBIT_PASS=rabbit - - OASIS_SERVER_DB_HOST=server-db - - OASIS_SERVER_DB_PASS=oasis - - OASIS_SERVER_DB_USER=oasis - - OASIS_SERVER_DB_NAME=oasis - - OASIS_SERVER_DB_PORT=5432 - - OASIS_SERVER_DB_ENGINE=django.db.backends.postgresql_psycopg2 - - OASIS_CELERY_DB_ENGINE=db+postgresql+psycopg2 - - OASIS_CELERY_DB_HOST=celery-db - - OASIS_CELERY_DB_PASS=password - - OASIS_CELERY_DB_USER=celery - - OASIS_CELERY_DB_NAME=celery - - OASIS_CELERY_DB_PORT=5432 + <<: *shared-env volumes: - filestore-OasisData:/shared-fs:rw - worker: + v2-task-controller: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} + command: [celery, -A, 'src.server.oasisapi.celery_app_v2', worker, --loglevel=INFO, -Q, task-controller] + links: + - server-db + - celery-db + - broker + environment: + <<: *shared-env + volumes: + - filestore-OasisData:/shared-fs:rw + celery-beat_v2: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${VERS_API:-latest} + command: [celery, -A, src.server.oasisapi.celery_app_v2, beat, --loglevel=INFO] + links: + - server-db + - celery-db + - broker + environment: + <<: *shared-env + volumes: *shared-volumes + v1-worker: restart: always image: ${WORKER_IMG:-coreoasis/model_worker}:${VERS_WORKER:-latest} + build: + context: . + dockerfile: Dockerfile.model_worker links: - celery-db - - rabbit:myrabbit + - broker:mybroker environment: - - OASIS_MODEL_SUPPLIER_ID=OasisLMF - - OASIS_MODEL_ID=PiWind - - OASIS_MODEL_VERSION_ID=1 - - OASIS_RABBIT_HOST=rabbit - - OASIS_RABBIT_PORT=5672 - - OASIS_RABBIT_USER=rabbit - - OASIS_RABBIT_PASS=rabbit - - OASIS_SERVER_DB_ENGINE=django.db.backends.postgresql_psycopg2 - - OASIS_CELERY_DB_ENGINE=db+postgresql+psycopg2 - - OASIS_CELERY_DB_HOST=celery-db - - OASIS_CELERY_DB_PASS=password - - OASIS_CELERY_DB_USER=celery - - OASIS_CELERY_DB_NAME=celery - - OASIS_CELERY_DB_PORT=5432 - - OASIS_MODEL_DATA_DIRECTORY=/home/worker/model + <<: *shared-env + OASIS_MODEL_SUPPLIER_ID: OasisLMF + OASIS_MODEL_ID: PiWind + OASIS_MODEL_VERSION_ID: 'v1' + OASIS_RUN_MODE: v1 + volumes: + - ./OasisPiWind/:/home/worker/model + - filestore-OasisData:/shared-fs:rw + v2-worker: + restart: always + image: ${WORKER_IMG:-coreoasis/model_worker}:${VERS_WORKER:-latest} + build: + context: . + dockerfile: Dockerfile.model_worker + links: + - celery-db + - broker:mybroker + environment: + <<: *shared-env + OASIS_MODEL_SUPPLIER_ID: OasisLMF + OASIS_MODEL_ID: PiWind + OASIS_MODEL_VERSION_ID: 'v2' + OASIS_RUN_MODE: v2 volumes: - ./OasisPiWind/:/home/worker/model - filestore-OasisData:/shared-fs:rw @@ -98,6 +163,8 @@ services: - POSTGRES_PASSWORD=oasis volumes: - server-db-OasisData:/var/lib/postgresql/data:rw + ports: + - 33307:3306 celery-db: restart: always image: postgres @@ -107,7 +174,9 @@ services: - POSTGRES_PASSWORD=password volumes: - celery-db-OasisData:/var/lib/postgresql/data:rw - rabbit: + ports: + - 33306:5432 + broker: restart: always image: rabbitmq:3.8.14-management environment: @@ -116,17 +185,8 @@ services: ports: - 5672:5672 - 15672:15672 -# flower: -# restart: always -# image: iserko/docker-celery-flower -# ports: -# - 5555:5555 -# environment: -# - CELERY_BROKER_URL=amqp://rabbit:rabbit@rabbit:5672 -# entrypoint: -# - flower -# - --port=5555 -# - --broker_api=http://rabbit:rabbit@rabbit:15672/api/ -# links: -# - celery-db -# - rabbit + channel-layer: + restart: always + image: redis:5.0.7 + ports: + - 6379:6379 diff --git a/oasis-ui-proxy.yml b/oasis-ui-proxy.yml index 63f18eb..2dd1c24 100644 --- a/oasis-ui-proxy.yml +++ b/oasis-ui-proxy.yml @@ -10,7 +10,7 @@ services: #- API_IP= - UI_DOCKER_IMAGE=coreoasis/oasisui_app:latest - API_PORT=8000 - - API_VERSION=v1 + - API_VERSION=v2 - API_SHARE_FILEPATH=./downloads - OASIS_ENVIRONMENT=oasis_localhost volumes: diff --git a/oasis-ui-standalone.yml b/oasis-ui-standalone.yml index 739f84a..11d1ec8 100644 --- a/oasis-ui-standalone.yml +++ b/oasis-ui-standalone.yml @@ -7,7 +7,7 @@ services: environment: - API_IP=server - API_PORT=8000 - - API_VERSION=v1 + - API_VERSION=v2 - API_SHARE_FILEPATH=./downloads - OASIS_ENVIRONMENT=oasis_localhost ports: diff --git a/portainer.yaml b/portainer.yaml index 6336ff8..eac2ad4 100644 --- a/portainer.yaml +++ b/portainer.yaml @@ -4,7 +4,7 @@ services: restart: always image: portainer/portainer:latest ports: - - 8001:8001 + - 8002:8002 - 9000:9000 volumes: - /var/run/docker.sock:/var/run/docker.sock