From 766bfa064edd0dd6181a7d9bf36fbdb5f827c745 Mon Sep 17 00:00:00 2001 From: Prasad Talasila Date: Sun, 3 Mar 2024 08:30:23 +0100 Subject: [PATCH] Restructures docker files (#577 and #561) - Adds authorization to docker local and trial deployments - Refactors development docker compose file to optionally include authorization - Improves READMEs for authorization --------- Co-authored-by: Astitva --- client/README.md | 1 + client/config/local.js | 19 ++ docker/.env | 9 + docker/README.md | 196 ++++++++++++------ docker/auth/README.md | 144 ------------- docker/compose.dev.yml | 44 ---- docker/compose.local.yml | 104 ++++++++-- docker/{auth => }/compose.server.yml | 13 +- docker/conf.local | 8 + docker/{auth/conf => conf.server} | 0 docker/{auth => dev}/.env | 2 +- docker/dev/README.md | 86 ++++++++ docker/{ => dev}/client.dockerfile | 0 .../compose.local.yml => dev/compose.dev.yml} | 13 +- docker/{ => dev}/libms.dockerfile | 0 docker/{auth => }/traefik-forward-auth.png | Bin docs/admin/servers/auth.md | 10 +- servers/config/gateway/README.md | 64 ------ servers/config/gateway/auth | 1 - .../gateway/dynamic/fileConfig.docker.yml | 48 ----- servers/config/gateway/dynamic/fileConfig.yml | 61 ------ servers/config/gateway/traefik.yml | 20 -- 22 files changed, 359 insertions(+), 484 deletions(-) create mode 100644 client/config/local.js create mode 100644 docker/.env delete mode 100644 docker/auth/README.md delete mode 100644 docker/compose.dev.yml rename docker/{auth => }/compose.server.yml (94%) create mode 100644 docker/conf.local rename docker/{auth/conf => conf.server} (100%) rename docker/{auth => dev}/.env (72%) create mode 100644 docker/dev/README.md rename docker/{ => dev}/client.dockerfile (100%) rename docker/{auth/compose.local.yml => dev/compose.dev.yml} (90%) rename docker/{ => dev}/libms.dockerfile (100%) rename docker/{auth => }/traefik-forward-auth.png (100%) delete mode 100644 servers/config/gateway/README.md delete mode 100644 servers/config/gateway/auth delete mode 100644 servers/config/gateway/dynamic/fileConfig.docker.yml delete mode 100644 servers/config/gateway/dynamic/fileConfig.yml delete mode 100644 servers/config/gateway/traefik.yml diff --git a/client/README.md b/client/README.md index a95ff4c69..1566627dd 100644 --- a/client/README.md +++ b/client/README.md @@ -22,6 +22,7 @@ yarn build #build the react app into build/ directory yarn develop #start the development server without building. Great for live edits. #Required: Choose one config for application (mandatory) +yarn config:local yarn config:dev yarn config:prod yarn config:test diff --git a/client/config/local.js b/client/config/local.js new file mode 100644 index 000000000..7c2fa66ce --- /dev/null +++ b/client/config/local.js @@ -0,0 +1,19 @@ +if (typeof window !== 'undefined') { + window.env = { + REACT_APP_ENVIRONMENT: 'dev', + REACT_APP_URL: 'http://localhost/', + REACT_APP_URL_BASENAME: '', + REACT_APP_URL_DTLINK: '/lab', + REACT_APP_URL_LIBLINK: '', + REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', + REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', + REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', + REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', + + REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', + REACT_APP_AUTH_AUTHORITY: 'https://gitlab.com/', + REACT_APP_REDIRECT_URI: 'http://localhost/Library', + REACT_APP_LOGOUT_REDIRECT_URI: 'http://localhost/', + REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', + }; +}; diff --git a/docker/.env b/docker/.env new file mode 100644 index 000000000..07a7962a7 --- /dev/null +++ b/docker/.env @@ -0,0 +1,9 @@ +DTAAS_DIR='/home/Desktop/DTaaS' +SERVER_DNS='foo.com' +OAUTH_URL='https://gitlab.foo.com' +CLIENT_ID='xx' +CLIENT_SECRET='xx' +OAUTH_SECRET='random-secret-string' +CLIENT_CONFIG='/home/Desktop/DTaaS/deploy/config/client/env.local.js' +username1='user1' +username2='user2' \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index f45489471..1b7b506e1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,104 +1,168 @@ -# Docker workflow for DTaaS +# Docker Compose for DTaaS with Backend Authorization -This readme will explain the building and use of different docker files -for use in development and installation of the DTaaS software. +This directory contains docker compose files for running the DTaaS with +backend authorization. The authorization happens at traefik using +[Traefik forward-auth](https://github.com/thomseddon/traefik-forward-auth). +The Traefik forward-auth uses OAuth2 for protecting all routes to user workspaces. -**NOTE**: A local docker and docker-compose installation is a pre-requisite -for using docker workflows. +## Design -## Folder Structure +An illustration of the docker containers used and the authorization +setup is shown here. -There are two dockerfiles for building the containers: +![Traefik OAuth](./traefik-forward-auth.png) -- **client.dockerfile**: Dockerfile for building - the client application container. -- **libms.dockerfile**: Dockerfile for building the library microservice container. +In the new application configuration, there are two OAuth2 applications. -There are also two compose files for development and local installation scenarios. +- The React single page application (SPA). The details of + this Oauth2 app are in + [client docs](../../docs/admin/client/auth.md). +- The Oauth2 server-side or web application for traefik forward-auth container. + The details are in [server docs](../../docs/admin/servers/auth.md). -- **compose.dev.yml:** Docker Compose configuration for development environment. -- **compose.local.yml:** Docker Compose configuration for localhost installation. +## Requirements -## Build and Publish Docker Images +The installation requirements to run this docker version of the DTaaS are: -### Users +- docker with compose plugin +- [gitlab oauth provider](https://docs.gitlab.com/ee/integration/oauth_provider.html#create-an-instance-wide-application) +- DNS name (optional, required only when the DTaaS is to be deployed on a web server) +- User accounts -Build and publish the docker images. This step is required only for -the publication of images to Docker Hub. This publishing step is managed -only by project maintainers. Regular users can skip this step. +### Create User Accounts -```sh -docker login -u -p -docker build -t intocps/libms:latest -f ./docker/libms.dockerfile . -docker tag intocps/libms:latest intocps/libms:version -docker push intocps/libms:latest -docker push intocps/libms:version +Create user accounts in gitlab for all the usernames mentioned in +`.env` and `conf` files. +The _trial_ installation script comes with two default +usernames - _user1_ and _user2_. For all other installation scenarios, +accounts with specific usernames need to be created on gitlab. -docker build -t intocps/dtaas-web:latest -f ./docker/client.dockerfile . -docker tag intocps/dtaas-web:latest intocps/dtaas-web:version -docker push intocps/dtaas-web:latest -docker push intocps/dtaas-web:version -``` +## Configuration -To tag version 0.3.1 for example, use +### Traefik forward-auth -```sh -docker tag intocps/dtaas-web:latest intocps/dtaas-web:0.3.1 -``` +The first step is to finish the configuration for +the traefik forward-auth container. +The details are in [server docs](../../docs/admin/servers/auth.md). -### Developers +### Docker Compose -Use of docker images is handy for developers as well. It is suggested -that developers build the required images locally on their computer and -use them for development purposes. The images can be built using +The docker compose configuration is in `.env`; it is a sample file. +It contains environment variables +that are used by the docker compose files. -```sh -docker-compose -f compose.dev.yml build -``` +Edit all the fields according to your specific case. + + | URL Path | Access Granted to |Access Granted to | + |:------------|:---------------|:---------------| + | DTAAS_DIR | '/home/Desktop/DTaaS' | Full path to the DTaaS directory. This is an absolute path with no trailing slash. | + | SERVER_DNS | _foo.com_ or _localhost_ | The server DNS, if you are deploying with a dedicated server. Remember not use http(s) at the beginning of the DNS string | + | BASE_URL | _gitlab.foo.com_ | The URL of your Gitlab instance | + | CLIENT_ID | 'xx' | The ID of your OAuth application | + | CLIENT_SECRET | 'xx' | The Secret of your OAuth application | + | OAUTH_SECRET | 'random-secret-string' | Any private random string | + | username1 | 'user1' | The gitlab instance username of a user of DTaaS | + | username2 | 'user2' | The gitlab instance username of a user of DTaaS | + | CLIENT_CONFIG | '/home/Desktop/DTaaS/deploy/config/client/env.js' | Full path to env.js file for client | + +Note: The Server DNS can also be an IP address. +However, for proper working it is neccessary to use the +same convention (IP/DNS) in the `CLIENT_CONFIG` file as well. + +### Website Client + +- Assign the correct client config file to `CLIENT_CONFIG` and + update the values. + + | Deployment Scenario | client env file | + |:-------|:------| + | localhost | deploy/config/client/env.local.js | + | trial | deploy/config/client/env.trial.js | + + This file has to be configured properly. + + Further explanation on the client configuration is available in + [client config page](../../docs/admin/client/CLIENT.md). + +### Lib MS + +No configuration change is needed for a general use case to bring up the LibMS. -## Running Docker Containers +In a specific case, +you can change the configuration of the LibMS by editing the file: -Follow these steps to use the application with docker. +- deploy/config/lib.docker -The DTaaS application requires multiple configuration files. The list of -configuration files to be modified are given for each scenario. +This is not recommended for a general use. +It should be done with care, making relevant changes +in all other related files, +and the compose file being used. -### Development Environment +### Caveat -This scenario is for software developers: +The usernames in the `.env` file need to match those in the conf file. +The conf.local is used by compose.local.yml and +conf.server is used by compose.server.yml. -The configuration files to be updated are: +Traefik routes are controlled by the `.env` file +Authentication on these routes is controlled by the `conf` file. +If a route is not specified in `conf` file but an authorisation is +requested by traefik for this unknown route, the default behavior of +traefik forward-auth kicks in. This default behavior is to enable +endpoint being available to any signed in user. -1. client/config/dev.js -1. deploy/config/lib.docker -1. servers/config/gateway/auth +If there are extra routes in `conf` file but these are not in `.env` file, +such routes are not served by traefik; it will give **404 server response**. -The relevant docker commands are: +## Run + +There are two usage scenarios, namely **localhost** and **trial** installation. + +Both the installation scenarios use the images already built and +deployed on docker hub, for the ML-workspace, client, and LibMS. +Both incorporate traefik forward-auth microservice for backend authorization. + +### Localhost + +The commands to start and stop the appliation are: ```bash -docker-compose -f compose.dev.yml up -d #start the application -docker-compose -f compose.dev.yml down #terminate the application +docker compose -f compose.local.yml --env-file .env up -d +docker compose -f compose.local.yml --env-file .env down ``` -### Localhost Use +To restart only a specific container, for example `client`` + +```bash +docker compose -f compose.local.yml --env-file .env up -d --force-recreate client +``` -This scenario is for users interested in using the software on -their computers (localhost): +## Trial Server Installation -The configuration files to be updated are: +The commands to start and stop the appliation are: -1. deploy/config/client/env.local.js -1. deploy/config/lib.docker -1. deploy/config/gateway/auth +```bash +docker compose -f compose.server.yml --env-file .env up -d +docker compose -f compose.server.yml --env-file .env down +``` -The relevant docker commands are: +To restart only a specific container, for example `client`` ```bash -docker-compose -f compose.local.yml up -d #start the application -docker-compose -f compose.local.yml down #terminate the application +docker compose -f compose.server.yml --env-file .env up -d --force-recreate client ``` -### Access the Application +## Use + +| Deployment Scenario | URL | +|:----|:----| +| localhost | _localhost_ | +| trial server | _foo.com_ | + +Sign in to gitlab instance with the your account. + +All the functionality of DTaaS should be available to you +through the single page client now. -You should access the application through the PORT mapped to the Traefik container. -e.g. `localhost:9000` +You may have to click Sign in to Gitlab on the Client page +and authorize access to the shown application. diff --git a/docker/auth/README.md b/docker/auth/README.md deleted file mode 100644 index ec4089725..000000000 --- a/docker/auth/README.md +++ /dev/null @@ -1,144 +0,0 @@ -# Docker Compose for DTaaS with Backend Authorization - -This directory contains docker compose files for running the DTaaS with -backend authorization. The authorization happens at traefik using -[Traefik forward-auth](https://github.com/thomseddon/traefik-forward-auth). -The Traefik forward-auth uses OAuth2 for protecting all routes to user workspaces. - -## Design - -An illustration of the docker containers used and the authorization -setup is shown here. - -![Traefik OAuth](./traefik-forward-auth.png) - -In the new application configuration, there are two OAuth2 applications. - -- The React single page application (SPA). The details of - this Oauth2 app are in - [client docs](../../docs/admin/client/auth.md). -- The Oauth2 server-side or web application for traefik forward-auth container. - The details are in [server docs](../../docs/admin/servers/auth.md). - -## Requirements - -The installation requirements to run this docker version of the DTaaS are: - -- docker with compose plugin -- [gitlab oauth provider](https://docs.gitlab.com/ee/integration/oauth_provider.html#create-an-instance-wide-application) -- DNS name (optional, required only when the DTaaS is to be deployed on a web server) -- User accounts - -### Create User Accounts - -Create user accounts in gitlab for all the usernames mentioned in -`.env` and `conf` files. -The _trial_ installation script comes with two default -usernames - _user1_ and _user2_. For all other installation scenarios, -accounts with specific usernames need to be created on gitlab. - -## Configuration - -### Traefik forward-auth - -The first step is to finish the configuration for -the traefik forward-auth container. -The details are in [server docs](../../docs/admin/servers/auth.md). - -### Docker Compose - -The docker compose configuration is in `.env`; it is a sample file. -It contains environment variables -that are used by the docker compose files. - -Edit all the fields according to your specific case. - - | URL Path | Access Granted to |Access Granted to | - |:------------|:---------------|:---------------| - | DTAAS_DIR | '/home/Desktop/DTaaS' | Full path to the DTaaS directory. This is an absolute path with no trailing slash. | - | SERVER_DNS | _foo.com_ or _localhost_ | The server DNS, if you are deploying with a dedicated server. Remember not use http(s) at the beginning of the DNS string | - | BASE_URL | _gitlab.foo.com_ | The URL of your Gitlab instance | - | CLIENT_ID | 'xx' | The ID of your OAuth application | - | CLIENT_SECRET | 'xx' | The Secret of your OAuth application | - | OAUTH_SECRET | 'random-secret-string' | Any private random string | - | username1 | 'user1' | The gitlab instance username of a user of DTaaS | - | username2 | 'user2' | The gitlab instance username of a user of DTaaS | - | ENV_JS_FILEPATH | '/home/Desktop/DTaaS/deploy/config/client/env.js' | Full path to env.js file for client | - -- Assign the correct client config file to `ENV_JS_FILEPATH and - update the values. - - | Deployment Scenario | client env file | - |:-------|:------| - | localhost | deploy/config/client/env.local.js | - | trial | deploy/config/client/env.trial.js | - - Further explanation on the client configuration is available in - [client config page](../../docs/admin/client/CLIENT.md). - -### Caveat - -The usernames in the `.env` file need to match those in the conf file. - -Traefik routes are controlled by the `.env` file -Authentication on these routes is controlled by the `conf` file. -If a route is not specified in `conf` file but an authorisation is -requested by traefik for this unknown route, the default behavior of -traefik forward-auth kicks in. This default behavior is to enable -endpoint being available to any signed in user. - -If there are extra routes in `conf` file but these are not in `.env` file, -such routes are not served by traefik; it will give **404 server response**. - -## Run - -There are two usage scenarios, namely **localhost** and **trial** installation. - -Both the installation scenarios use the images already built and -deployed on docker hub, for the ML-workspace, client, and LibMS. -Both incorporate traefik forward-auth microservice for backend authorization. - -### Localhost - -The commands to start and stop the appliation are: - -```bash -docker compose -f compose.local.yml --env-file .env up -d -docker compose -f compose.local.yml --env-file .env down -``` - -To restart only a specific container, for example `client`` - -```bash -docker compose -f compose.local.yml --env-file .env up -d --force-recreate client -``` - -## Trial Server Installation - -The commands to start and stop the appliation are: - -```bash -docker compose -f compose.server.yml --env-file .env up -d -docker compose -f compose.server.yml --env-file .env down -``` - -To restart only a specific container, for example `client`` - -```bash -docker compose -f compose.server.yml --env-file .env up -d --force-recreate client -``` - -## Use - -| Deployment Scenario | URL | -|:----|:----| -| localhost | _localhost_ | -| trial server | _foo.com_ | - -Sign in to gitlab instance with the your account. - -All the functionality of DTaaS should be available to you -through the single page client now. - -You may have to click Sign in to Gitlab on the Client page -and authorize access to the shown application. diff --git a/docker/compose.dev.yml b/docker/compose.dev.yml deleted file mode 100644 index 8751ecf06..000000000 --- a/docker/compose.dev.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '3' - -services: - client: - build: - context: ../ - dockerfile: ./docker/client.dockerfile - ports: - - "4000:4000" - volumes: - - "../client/config/dev.js:/dtaas/client/build/env.js" - libms: - build: - context: ../ - dockerfile: ./docker/libms.dockerfile - ports: - - "4001:4001" - volumes: - - "../deploy/config/lib.docker:/dtaas/libms/.env" - - "../files:/dtaas/libms/files" - - ml-workspace-user1: - image: mltooling/ml-workspace-minimal:0.13.2 - container_name: ml-workspace-user1 - ports: - - "8090:8080" - volumes: - - "../files/user1:/workspace" - - "../files/common:/workspace/common" - environment: - - AUTHENTICATE_VIA_JUPYTER - - WORKSPACE_BASE_URL=user1 - shm_size: 512m - - traefik-gateway: - image: traefik:v2.10 - container_name: traefik-gateway - ports: - - "9000:80" - volumes: - - "../servers/config/gateway/traefik.yml:/etc/traefik/traefik.yml" - - "../servers/config/gateway/auth:/etc/traefik/auth" - - "../servers/config/gateway/dynamic/fileConfig.docker.yml:/etc/traefik/dynamic/fileConfig.yml" - - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file diff --git a/docker/compose.local.yml b/docker/compose.local.yml index bceccafbc..88c212671 100644 --- a/docker/compose.local.yml +++ b/docker/compose.local.yml @@ -1,34 +1,94 @@ version: '3' - services: - client: - image: intocps/dtaas-web + traefik: + image: traefik:v2.10 + command: + - "--api.insecure=true" + - "--providers.docker=true" + - "--entryPoints.web.address=:80" + - "--entrypoints.web.forwardedHeaders.insecure=true" + - "--entrypoints.web.proxyProtocol.insecure=true" + ports: + - "80:80" volumes: - - "../deploy/config/client/env.local.js:/dtaas/client/build/env.js" + - /var/run/docker.sock:/var/run/docker.sock + libms: - image: intocps/libms + image: intocps/libms:latest volumes: - - "../deploy/config/lib.docker:/dtaas/libms/.env" - - "../files:/dtaas/libms/files" + - ${DTAAS_DIR}/deploy/config/lib.docker:/dtaas/libms/.env + - ${DTAAS_DIR}/files:/dtaas/libms/files + labels: + - "traefik.enable=true" + - "traefik.http.routers.libms.entryPoints=web" + - "traefik.http.services.libms.loadbalancer.server.port=4001" + - "traefik.http.routers.libms.rule= PathPrefix(`/lib`)" + - "traefik.http.routers.libms.middlewares=traefik-forward-auth" - ml-workspace-user1: + client: + image: intocps/dtaas-web:latest + volumes: + - ${CLIENT_CONFIG}:/dtaas/client/build/env.js + labels: + - "traefik.enable=true" + - "traefik.http.routers.client.entryPoints=web" + - "traefik.http.services.client.loadbalancer.server.port=4000" + - "traefik.http.routers.client.middlewares=traefik-forward-auth" + - "traefik.http.routers.client.rule=PathPrefix(`/`)" + + user1: image: mltooling/ml-workspace-minimal:0.13.2 - container_name: ml-workspace-user1 volumes: - - "../files/user1:/workspace" - - "../files/common:/workspace/common" + - ${DTAAS_DIR}/files/common:/workspace/common + - ${DTAAS_DIR}/files/${username1}:/workspace environment: - - AUTHENTICATE_VIA_JUPYTER - - WORKSPACE_BASE_URL=user1 + - AUTHENTICATE_VIA_JUPYTER= + - WORKSPACE_BASE_URL=${username1} shm_size: 512m + labels: + - "traefik.enable=true" + - "traefik.http.routers.u1.entryPoints=web" + - "traefik.http.routers.u1.rule=PathPrefix(`/${username1}`) " + - "traefik.http.routers.u1.middlewares=traefik-forward-auth" + + user2: + image: mltooling/ml-workspace-minimal:0.13.2 + volumes: + - ${DTAAS_DIR}/files/common:/workspace/common + - ${DTAAS_DIR}/files/${username2}:/workspace + environment: + - AUTHENTICATE_VIA_JUPYTER= + - WORKSPACE_BASE_URL=${username2} + shm_size: 512m + labels: + - "traefik.enable=true" + - "traefik.http.routers.u2.entryPoints=web" + - "traefik.http.routers.u2.rule=PathPrefix(`/${username2}`) " + - "traefik.http.routers.u2.middlewares=traefik-forward-auth" - traefik-gateway: - image: traefik:v2.10 - container_name: traefik-gateway - ports: - - "9000:80" + traefik-forward-auth: + image: thomseddon/traefik-forward-auth:latest volumes: - - "../deploy/config/gateway/traefik.yml:/etc/traefik/traefik.yml" - - "../deploy/config/gateway/auth:/etc/traefik/auth" - - "../deploy/config/gateway/dynamic/fileConfig.docker.yml:/etc/traefik/dynamic/fileConfig.yml" - - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file + - ${DTAAS_DIR}/docker/conf.local:/conf + environment: + - DEFAULT_PROVIDER=generic-oauth + - PROVIDERS_GENERIC_OAUTH_AUTH_URL=${OAUTH_URL}/oauth/authorize + - PROVIDERS_GENERIC_OAUTH_TOKEN_URL=${OAUTH_URL}/oauth/token + - PROVIDERS_GENERIC_OAUTH_USER_URL=${OAUTH_URL}/api/v4/user + - PROVIDERS_GENERIC_OAUTH_CLIENT_ID=${CLIENT_ID} + - PROVIDERS_GENERIC_OAUTH_CLIENT_SECRET=${CLIENT_SECRET} + - PROVIDERS_GENERIC_OAUTH_SCOPE=read_user + - SECRET= ${OAUTH_SECRET} + # INSECURE_COOKIE is required if not using a https entrypoint + - INSECURE_COOKIE=true + - CONFIG=/conf + labels: + - "traefik.enable=true" + + - "traefik.http.routers.redirect.entryPoints=web" + - "traefik.http.routers.redirect.rule=PathPrefix(`/_oauth`)" + - "traefik.http.routers.redirect.middlewares=traefik-forward-auth" + + - "traefik.http.middlewares.traefik-forward-auth.forwardauth.address=http://traefik-forward-auth:4181" + - "traefik.http.middlewares.traefik-forward-auth.forwardauth.authResponseHeaders=X-Forwarded-User" + - "traefik.http.services.traefik-forward-auth.loadbalancer.server.port=4181" diff --git a/docker/auth/compose.server.yml b/docker/compose.server.yml similarity index 94% rename from docker/auth/compose.server.yml rename to docker/compose.server.yml index 42dec4c2f..9848d088a 100644 --- a/docker/auth/compose.server.yml +++ b/docker/compose.server.yml @@ -24,11 +24,11 @@ services: - "traefik.http.services.libms.loadbalancer.server.port=4001" - "traefik.http.routers.libms.rule=Host(`${SERVER_DNS}`)&&PathPrefix(`/lib`)" - "traefik.http.routers.libms.middlewares=traefik-forward-auth" - + client: image: intocps/dtaas-web:latest volumes: - - ${ENV_JS_FILEPATH}:/dtaas/client/build/env.js + - ${CLIENT_CONFIG}:/dtaas/client/build/env.js labels: - "traefik.enable=true" - "traefik.http.routers.client.entryPoints=web" @@ -39,11 +39,12 @@ services: user1: image: mltooling/ml-workspace-minimal:0.13.2 volumes: - - ${DTAAS_DIR}/files/common:/workspace/common + - ${DTAAS_DIR}/files/common:/workspace/common - ${DTAAS_DIR}/files/${username1}:/workspace environment: - AUTHENTICATE_VIA_JUPYTER= - WORKSPACE_BASE_URL=${username1} + shm_size: 512m labels: - "traefik.enable=true" - "traefik.http.routers.u1.entryPoints=web" @@ -58,17 +59,17 @@ services: environment: - AUTHENTICATE_VIA_JUPYTER= - WORKSPACE_BASE_URL=${username2} + shm_size: 512m labels: - "traefik.enable=true" - "traefik.http.routers.u2.entryPoints=web" - "traefik.http.routers.u2.rule=Host(`${SERVER_DNS}`)&&PathPrefix(`/${username2}`)" - "traefik.http.routers.u2.middlewares=traefik-forward-auth" - - + traefik-forward-auth: image: thomseddon/traefik-forward-auth:latest volumes: - - ${DTAAS_DIR}/docker/auth/conf:/conf + - ${DTAAS_DIR}/docker/conf.server:/conf environment: - DEFAULT_PROVIDER=generic-oauth - PROVIDERS_GENERIC_OAUTH_AUTH_URL=${OAUTH_URL}/oauth/authorize diff --git a/docker/conf.local b/docker/conf.local new file mode 100644 index 000000000..971d212a3 --- /dev/null +++ b/docker/conf.local @@ -0,0 +1,8 @@ +rule.libms.action=allow +rule.libms.rule=PathPrefix(`/lib`) + +rule.onlyu1.action=allow +rule.onlyu1.rule=PathPrefix(`/user1`) + +rule.onlyu2.action=allow +rule.onlyu2.rule=PathPrefix(`/user2`) \ No newline at end of file diff --git a/docker/auth/conf b/docker/conf.server similarity index 100% rename from docker/auth/conf rename to docker/conf.server diff --git a/docker/auth/.env b/docker/dev/.env similarity index 72% rename from docker/auth/.env rename to docker/dev/.env index 871186875..80b1ec640 100644 --- a/docker/auth/.env +++ b/docker/dev/.env @@ -4,6 +4,6 @@ OAUTH_URL='https://gitlab.foo.com' CLIENT_ID='xx' CLIENT_SECRET='xx' OAUTH_SECRET='random-secret-string' -ENV_JS_FILEPATH='/home/Desktop/DTaaS/deploy/config/client/env.local.js' +OAUTH_CONF_FILEPATH='/home/Desktop/DTaaS/docker/conf.local' username1='user1' username2='user2' diff --git a/docker/dev/README.md b/docker/dev/README.md new file mode 100644 index 000000000..450667fa7 --- /dev/null +++ b/docker/dev/README.md @@ -0,0 +1,86 @@ +# Docker workflow for DTaaS + +This readme will explain the building and use of different docker files +for use in development and installation of the DTaaS software. + +**NOTE**: A local docker and docker-compose installation is a pre-requisite +for using docker workflows. + +## Folder Structure + +There are two dockerfiles for building the containers: + +- **client.dockerfile**: Dockerfile for building + the client application container. +- **libms.dockerfile**: Dockerfile for building the library microservice container. + +There is a specific compose file for development: + +- **compose.dev.yml:** Docker Compose configuration for development environment. + +## Build and Publish Docker Images + +### Users + +Build and publish the docker images. This step is required only for +the publication of images to Docker Hub. This publishing step is managed +only by project maintainers. Regular users can skip this step. + +```sh +docker login -u -p +docker build -t intocps/libms:latest -f ./docker/libms.dockerfile . +docker tag intocps/libms:latest intocps/libms:version +docker push intocps/libms:latest +docker push intocps/libms:version + +docker build -t intocps/dtaas-web:latest -f ./docker/client.dockerfile . +docker tag intocps/dtaas-web:latest intocps/dtaas-web:version +docker push intocps/dtaas-web:latest +docker push intocps/dtaas-web:version +``` + +To tag version 0.3.1 for example, use + +```sh +docker tag intocps/dtaas-web:latest intocps/dtaas-web:0.3.1 +``` + +### Developers + +Use of docker images is handy for developers as well. It is suggested +that developers build the required images locally on their computer and +use them for development purposes. The images can be built using + +```sh +docker-compose -f compose.dev.yml build +``` + +## Running Docker Containers + +Follow these steps to use the application with docker. + +The DTaaS application requires multiple configuration files. The list of +configuration files to be modified are given for each scenario. + +### Development Environment + +This scenario is for software developers: + +### Client Website & Lib Configuration + +The configuration files to be updated are: + +1. client/config/local.js +1. deploy/config/lib.docker (optional) + +The relevant docker commands are: + +```bash +docker-compose -f compose.dev.yml up -d #start the application +docker-compose -f compose.dev.yml down #terminate the application +``` + +### Access the Application + +You should access the application through the PORT mapped to the Traefik container. +e.g. `localhost` diff --git a/docker/client.dockerfile b/docker/dev/client.dockerfile similarity index 100% rename from docker/client.dockerfile rename to docker/dev/client.dockerfile diff --git a/docker/auth/compose.local.yml b/docker/dev/compose.dev.yml similarity index 90% rename from docker/auth/compose.local.yml rename to docker/dev/compose.dev.yml index 18af132a3..02f206a9f 100644 --- a/docker/auth/compose.local.yml +++ b/docker/dev/compose.dev.yml @@ -14,9 +14,11 @@ services: - /var/run/docker.sock:/var/run/docker.sock client: - image: intocps/dtaas-web:latest + build: + context: ${DTAAS_DIR}/ + dockerfile: ${DTAAS_DIR}/docker/dev/client.dockerfile volumes: - - ${ENV_JS_FILEPATH}:/dtaas/client/build/env.js + - "${DTAAS_DIR}/client/config/local.js:/dtaas/client/build/env.js" labels: - "traefik.enable=true" - "traefik.http.routers.client.entryPoints=web" @@ -26,6 +28,9 @@ services: libms: image: intocps/libms:latest + build: + context: ${DTAAS_DIR}/ + dockerfile: ${DTAAS_DIR}/docker/libms.dockerfile volumes: - ${DTAAS_DIR}/deploy/config/lib.docker:/dtaas/libms/.env - ${DTAAS_DIR}/files:/dtaas/libms/files @@ -44,6 +49,7 @@ services: environment: - AUTHENTICATE_VIA_JUPYTER= - WORKSPACE_BASE_URL=${username1} + shm_size: 512m labels: - "traefik.enable=true" - "traefik.http.routers.u1.entryPoints=web" @@ -58,6 +64,7 @@ services: environment: - AUTHENTICATE_VIA_JUPYTER= - WORKSPACE_BASE_URL=${username2} + shm_size: 512m labels: - "traefik.enable=true" - "traefik.http.routers.u2.entryPoints=web" @@ -67,7 +74,7 @@ services: traefik-forward-auth: image: thomseddon/traefik-forward-auth:latest volumes: - - ${DTAAS_DIR}/docker/auth/conf:/conf + - ${OAUTH_CONF_FILEPATH}:/conf environment: - DEFAULT_PROVIDER=generic-oauth - PROVIDERS_GENERIC_OAUTH_AUTH_URL=${OAUTH_URL}/oauth/authorize diff --git a/docker/libms.dockerfile b/docker/dev/libms.dockerfile similarity index 100% rename from docker/libms.dockerfile rename to docker/dev/libms.dockerfile diff --git a/docker/auth/traefik-forward-auth.png b/docker/traefik-forward-auth.png similarity index 100% rename from docker/auth/traefik-forward-auth.png rename to docker/traefik-forward-auth.png diff --git a/docs/admin/servers/auth.md b/docs/admin/servers/auth.md index 3566f869c..a478e1695 100644 --- a/docs/admin/servers/auth.md +++ b/docs/admin/servers/auth.md @@ -42,11 +42,13 @@ Here are the steps to get started: - The callback URL informs the OAuth provider of the page where signed-in users should be redirected. It is the landing - homepage of the DTaaS applicati1on. - (either or ) + homepage of the DTaaS application. + (either or ) - The logout URL is the URL for signout of gitlab and clear authorization within traefik-forward auth. - (either or ) + (either or ). + The logout URL is to help users logout of traefik forward-auth. The logout + URL should not be entered into Gitlab OAuth application setup. **4. OAuth Application Creation:** @@ -72,7 +74,7 @@ Here are the steps to get started: |Application Secret|CLIENT_SECRET| _xx_ | |Callback URL|(to be directly entered in Gitlab OAuth registration)|| |Forward-auth secret|OAUTH_SECRET|_random-secret-string_ (password for forward-auth, can be changed to your preferred string) | -|Scopes|email|| +|Scopes| read_user || ## Development Environment diff --git a/servers/config/gateway/README.md b/servers/config/gateway/README.md deleted file mode 100644 index 204e61a5a..000000000 --- a/servers/config/gateway/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# The gateway server - -Run the Traefik gateway server in HTTP mode to experience the DTaaS application. -HTTPS mode is disabled for now. - -## The background services - -The gateway requires background services to serve the URLs. These background -services must be running in order for the gateway to service user requests. -The default configuration uses two services at the following URLs: - -| Route / URL | Background Service | Service URL | -| :-------------- | :----------------- | :------------- | -| localhost | React Website | localhost:4000 | -| localhost/lib | Lib Microservice | localhost:4001 | -| localhost/user1 | ML Workspace | localhost:8090 | -| localhost/user2 | ML Workspace | localhost:8091 | -| | - -## Start the Gateway - -```bash -docker run -d \ - --name "traefik-gateway" \ - --network=host -v "$PWD/traefik.yml:/etc/traefik/traefik.yml" \ - -v "$PWD/auth:/etc/traefik/auth" \ - -v "$PWD/dynamic:/etc/traefik/dynamic" \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --restart always \ - traefik:v2.10 -``` - -## Authorization - -The dummy username is `foo` and the password is `bar`. -Please change this before starting the gateway. - -```bash -rm auth -htpasswd -c auth -password: -``` - -The change in password becomes effective upon restart of **traefik-gateway** container. - -## Update Configuration - -The gateway serves routes specified in _dynamic/fileConfig.yml_ file. -The **traefik-gateway** gateway comes with ability to receive dynamic configuration. -You can update the configuration in this file to reflect your local setup. -See [Traefik help docs](https://doc.traefik.io/traefik/providers/file/) -for more information. - -The routes / URLs need to be updated for your local setup. -The current version of software only works for non-localhost -setting, i.e. URL other than the localhost. Here is an example, - -| Route / URL | Background Service | Service URL | -| :------------ | :----------------- | :------------- | -| foo.com | React Website | localhost:4000 | -| foo.com/lib | Lib Microservice | localhost:4001 | -| foo.com/user1 | ML Workspace | localhost:8090 | -| foo.com/user2 | ML Workspace | localhost:8091 | -| | diff --git a/servers/config/gateway/auth b/servers/config/gateway/auth deleted file mode 100644 index 2e468b945..000000000 --- a/servers/config/gateway/auth +++ /dev/null @@ -1 +0,0 @@ -foo:$apr1$fGdMVLcl$dMzJW.JF8Rn6Vzrf5uxaU/ diff --git a/servers/config/gateway/dynamic/fileConfig.docker.yml b/servers/config/gateway/dynamic/fileConfig.docker.yml deleted file mode 100644 index 359314a0b..000000000 --- a/servers/config/gateway/dynamic/fileConfig.docker.yml +++ /dev/null @@ -1,48 +0,0 @@ -http: - routers: - dtaas: - entryPoints: - - http - rule: 'Host(`localhost`)' - middlewares: - - basic-auth - service: dtaas - - user1: - entryPoints: - - http - rule: 'Host(`localhost`) && PathPrefix(`/user1`)' - middlewares: - - basic-auth - service: user1 - - libms: - entryPoints: - - http - rule: 'Host(`localhost`) && PathPrefix(`/lib`)' - service: libms - - - # Middleware: Basic authentication - middlewares: - basic-auth: - basicAuth: - usersFile: "/etc/traefik/auth" - removeHeader: true - - - services: - dtaas: - loadBalancer: - servers: - - url: "http://client:4000" - - user1: - loadBalancer: - servers: - - url: "http://ml-workspace-user1:8080" - - libms: - loadBalancer: - servers: - - url: "http://libms:4001" \ No newline at end of file diff --git a/servers/config/gateway/dynamic/fileConfig.yml b/servers/config/gateway/dynamic/fileConfig.yml deleted file mode 100644 index 86c2d0ba7..000000000 --- a/servers/config/gateway/dynamic/fileConfig.yml +++ /dev/null @@ -1,61 +0,0 @@ -http: - routers: - dtaas: - entryPoints: - - http - rule: 'Host(`localhost`)' - middlewares: - - basic-auth - service: dtaas - - user1: - entryPoints: - - http - rule: 'Host(`localhost`) && PathPrefix(`/user1`)' - middlewares: - - basic-auth - service: user1 - - user2: - entryPoints: - - http - rule: 'Host(`localhost`) && PathPrefix(`/user2`)' - middlewares: - - basic-auth - service: user2 - - libms: - entryPoints: - - http - rule: 'Host(`localhost`) && PathPrefix(`/lib`)' - service: libms - - - # Middleware: Basic authentication - middlewares: - basic-auth: - basicAuth: - usersFile: "/etc/traefik/auth" - removeHeader: true - - - services: - dtaas: - loadBalancer: - servers: - - url: "http://localhost:4000" - - user1: - loadBalancer: - servers: - - url: "http://localhost:8090" - - user2: - loadBalancer: - servers: - - url: "http://localhost:8091" - - libms: - loadBalancer: - servers: - - url: "http://localhost:4001" \ No newline at end of file diff --git a/servers/config/gateway/traefik.yml b/servers/config/gateway/traefik.yml deleted file mode 100644 index 4f79c1d6f..000000000 --- a/servers/config/gateway/traefik.yml +++ /dev/null @@ -1,20 +0,0 @@ -entryPoints: - http: - address: :80 - -providers: - providersThrottleDuration: 2s - - # File provider for connecting things that are outside of docker / defining middleware - file: - filename: /etc/traefik/dynamic/fileConfig.yml - watch: true - -# Enable traefik ui -#dapi: -# dashboard: true -# insecure: true - -# Log level INFO|DEBUG|ERROR -log: - level: DEBUG