diff --git a/cli/README.md b/cli/README.md index cd15630a9..2ccdc3134 100644 --- a/cli/README.md +++ b/cli/README.md @@ -84,8 +84,8 @@ This brings up the containers, without the AuthMS authentication. - Currently the _email_ fields for each user in _dtaas.toml_ are not in use, and are not necessary to fill in. These emails must be configured manually - for each user in the docker/conf.local or - docker/conf.server files and the _traefik-forward-auth_ + for each user in the + deploy/docker/conf.server files and the _traefik-forward-auth_ container must be restarted. This is done as follows: - Go to the _docker_ directory @@ -94,13 +94,6 @@ This brings up the containers, without the AuthMS authentication. cd /docker ``` -- Add two lines to the `conf.local` file - -```txt -rule.onlyu4.action=allow -rule.onlyu4.rule=PathPrefix(`/user4`) -``` - - Add three lines to the `conf.server` file ```txt @@ -109,16 +102,12 @@ rule.onlyu3.rule=PathPrefix(`/user3`) rule.onlyu3.whitelist = user3@emailservice.com ``` -Run the appropritate command for a server/local installation: +Run the command for these changes to take effect: ```bash docker compose -f compose.server.yml --env-file .env up -d --force-recreate traefik-forward-auth ``` -```bash -docker compose -f compose.local.yml --env-file .env up -d --force-recreate traefik-forward-auth -``` - The new users are now added to the DTaaS instance, with authorization enabled. diff --git a/deploy/config/client/env.trial.js b/deploy/config/client/env.trial.js deleted file mode 100644 index 8b8f64c5a..000000000 --- a/deploy/config/client/env.trial.js +++ /dev/null @@ -1,19 +0,0 @@ -if (typeof window !== 'undefined') { - window.env = { - REACT_APP_ENVIRONMENT: 'trial', - REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: '', - REACT_APP_URL_DTLINK: '/lab', - REACT_APP_URL_LIBLINK: '', - REACT_APP_WORKBENCHLINK_VNCDESKTOP: '/tools/vnc/?password=vncpassword', - REACT_APP_WORKBENCHLINK_VSCODE: '/tools/vscode/', - REACT_APP_WORKBENCHLINK_JUPYTERLAB: '/lab', - REACT_APP_WORKBENCHLINK_JUPYTERNOTEBOOK: '', - - REACT_APP_CLIENT_ID: '934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0', - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/', - REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api', - }; -}; diff --git a/deploy/config/gateway/auth b/deploy/config/gateway/auth deleted file mode 100644 index 2e468b945..000000000 --- a/deploy/config/gateway/auth +++ /dev/null @@ -1 +0,0 @@ -foo:$apr1$fGdMVLcl$dMzJW.JF8Rn6Vzrf5uxaU/ diff --git a/deploy/config/gateway/fileConfig.local.yml b/deploy/config/gateway/fileConfig.local.yml deleted file mode 100644 index b07b2b5a2..000000000 --- a/deploy/config/gateway/fileConfig.local.yml +++ /dev/null @@ -1,35 +0,0 @@ -http: - routers: - dtaas: - entryPoints: - - http - rule: 'Host(`localhost`)' - service: dtaas - - user1: - entryPoints: - - http - rule: 'Host(`localhost`) && PathPrefix(`/user1`)' - service: user1 - - libms: - entryPoints: - - http - rule: 'Host(`localhost`) && PathPrefix(`/lib`)' - service: libms - - services: - dtaas: - loadBalancer: - servers: - - url: "http://localhost:4000" - - user1: - loadBalancer: - servers: - - url: "http://localhost:8090" - - libms: - loadBalancer: - servers: - - url: "http://localhost:4001" diff --git a/deploy/config/gateway/fileConfig.trial.yml b/deploy/config/gateway/fileConfig.trial.yml deleted file mode 100644 index b6944f72d..000000000 --- a/deploy/config/gateway/fileConfig.trial.yml +++ /dev/null @@ -1,48 +0,0 @@ -http: - routers: - dtaas: - entryPoints: - - http - rule: 'Host(`foo.com`)' - middlewares: - - basic-auth - service: dtaas - - user1: - entryPoints: - - http - rule: 'Host(`foo.com`) && PathPrefix(`/user1`)' - middlewares: - - basic-auth - service: user1 - - libms: - entryPoints: - - http - rule: 'Host(`foo.com`) && PathPrefix(`/lib`)' - service: libms - - - # Middleware: Basic authentication - middlewares: - basic-auth: - basicAuth: - usersFile: "/etc/traefik/auth" - removeHeader: true - - - services: - dtaas: - loadBalancer: - servers: - - url: "http://localhost:4000" - - user1: - loadBalancer: - servers: - - url: "http://localhost:8090" - - libms: - loadBalancer: - servers: - - url: "http://localhost:4001" \ No newline at end of file diff --git a/deploy/config/gateway/fileConfig.yml b/deploy/config/gateway/fileConfig.yml deleted file mode 100644 index efd35bb28..000000000 --- a/deploy/config/gateway/fileConfig.yml +++ /dev/null @@ -1,61 +0,0 @@ -http: - routers: - dtaas: - entryPoints: - - http - rule: 'Host(`foo.com`)' - middlewares: - - basic-auth - service: dtaas - - user1: - entryPoints: - - http - rule: 'Host(`foo.com`) && PathPrefix(`/user1`)' - middlewares: - - basic-auth - service: user1 - - user2: - entryPoints: - - http - rule: 'Host(`foo.com`) && PathPrefix(`/user2`)' - middlewares: - - basic-auth - service: user2 - - libms: - entryPoints: - - http - rule: 'Host(`foo.com`) && PathPrefix(`/lib`)' - service: libms - - - # Middleware: Basic authentication - middlewares: - basic-auth: - basicAuth: - usersFile: "/etc/traefik/auth" - removeHeader: true - - - services: - dtaas: - loadBalancer: - servers: - - url: "http://localhost:4000" - - user1: - loadBalancer: - servers: - - url: "http://localhost:8090" - - user2: - loadBalancer: - servers: - - url: "http://localhost:8091" - - libms: - loadBalancer: - servers: - - url: "http://localhost:4001" \ No newline at end of file diff --git a/deploy/config/gateway/traefik.yml b/deploy/config/gateway/traefik.yml deleted file mode 100644 index 4f79c1d6f..000000000 --- a/deploy/config/gateway/traefik.yml +++ /dev/null @@ -1,20 +0,0 @@ -entryPoints: - http: - address: :80 - -providers: - providersThrottleDuration: 2s - - # File provider for connecting things that are outside of docker / defining middleware - file: - filename: /etc/traefik/dynamic/fileConfig.yml - watch: true - -# Enable traefik ui -#dapi: -# dashboard: true -# insecure: true - -# Log level INFO|DEBUG|ERROR -log: - level: DEBUG diff --git a/docs/admin/cli.md b/docs/admin/cli.md index 36830239a..49b5f0791 100644 --- a/docs/admin/cli.md +++ b/docs/admin/cli.md @@ -81,8 +81,8 @@ This brings up the containers, without the AuthMS authentication. - Currently the _email_ fields for each user in _dtaas.toml_ are not in use, and are not necessary to fill in. These emails must be configured manually - for each user in the docker/conf.local or - docker/conf.server files and the _traefik-forward-auth_ + for each user in the + deploy/docker/conf.server files and the _traefik-forward-auth_ container must be restarted. This is done as follows: - Go to the _docker_ directory @@ -91,13 +91,6 @@ This brings up the containers, without the AuthMS authentication. cd /deploy/docker ``` -- Add two lines to the `conf.local` file - -```txt -rule.onlyu4.action=allow -rule.onlyu4.rule=PathPrefix(`/user4`) -``` - - Add three lines to the `conf.server` file ```txt @@ -106,16 +99,12 @@ rule.onlyu3.rule=PathPrefix(`/user3`) rule.onlyu3.whitelist = user3@emailservice.com ``` -Run the appropritate command for a server/local installation: +Run the appropritate command for a server installation: ```bash docker compose -f compose.server.yml --env-file .env up -d --force-recreate traefik-forward-auth ``` -```bash -docker compose -f compose.local.yml --env-file .env up -d --force-recreate traefik-forward-auth -``` - The new users are now added to the DTaaS instance, with authorization enabled. diff --git a/docs/admin/client/auth.md b/docs/admin/client/auth.md index 9011ddb4e..409f61af2 100644 --- a/docs/admin/client/auth.md +++ b/docs/admin/client/auth.md @@ -1,11 +1,5 @@ # OAuth for React Client - - -!!! failure - Add screenshots from dtaas-digitaltwin.com demo. - - To enable user authorization on DTaaS React client website, you will use the OAuth authorization protocol, specifically the PKCE authorization flow. Here are the steps to get started: @@ -40,12 +34,16 @@ Here are the steps to get started: - During the creation of the OAuth application on GitLab, you need to specify the scope. Choose openid, profile, read_user, read_repository, and api scopes. + ![Creation of Client OAuth Application](client-oauth-name.png) + **5. Application ID:** - After successfully creating the OAuth application, GitLab generates an application ID. This is a long string of HEX values that you will need for your configuration files. + ![Scopes for Client OAuth Application](client-oauth-scopes.png) + **6. Required Information from OAuth Application:** - You will need the following information from the OAuth application @@ -58,26 +56,11 @@ Here are the steps to get started: |Callback URL|REACT_APP_REDIRECT_URI|[https://foo.com/Library](https://foo.com/Library)| |Scopes|REACT_APP_GITLAB_SCOPES|openid, profile, read_user, read_repository, api| + ![Summary for Client OAuth Application](client-oauth-id.png) + **7. Create User Accounts:** Create user accounts in gitlab for all the usernames chosen during installation. The _trial_ installation script comes with two default usernames - _user1_ and _user2_. For all other installation scenarios, accounts with specific usernames need to be created on gitlab. - -## Development Environment - -There needs to be a valid callback and logout URLs for development and testing -purposes. You can use the same oauth application id for both development, testing -and deployment scenarios. Only the callback and logout URLs change. It is possible -to register multiple callback URLs in one oauth application. In order to use oauth -for development and testing on developer computer (localhost), you need to add the -following to oauth callback URL. - -```txt -DTaaS application URL: http://localhost:4000 -Callback URL: http://localhost:4000/Library -Logout URL: http://localhost:4000 -``` - -The port 4000 is the default port for running the client website. diff --git a/docs/admin/client/client-oauth-id.png b/docs/admin/client/client-oauth-id.png new file mode 100644 index 000000000..a509b60c7 Binary files /dev/null and b/docs/admin/client/client-oauth-id.png differ diff --git a/docs/admin/client/client-oauth-name.png b/docs/admin/client/client-oauth-name.png new file mode 100644 index 000000000..fa9ed2698 Binary files /dev/null and b/docs/admin/client/client-oauth-name.png differ diff --git a/docs/admin/client/client-oauth-scopes.png b/docs/admin/client/client-oauth-scopes.png new file mode 100644 index 000000000..8c879da36 Binary files /dev/null and b/docs/admin/client/client-oauth-scopes.png differ diff --git a/docs/admin/guides/add_user.md b/docs/admin/guides/add_user.md index 860eef22f..353d687a6 100644 --- a/docs/admin/guides/add_user.md +++ b/docs/admin/guides/add_user.md @@ -1,81 +1,41 @@ -# Add a new user +# Add User -This page will guide you on, how to add more users to the DTaas. Please do the following: +This page provides steps to adding a user from a DTaaS installation. +The username **alice** is used here to illustrate the steps involved in +removing a user account. - -!!! important - Make sure to replace **** and **** - Select a port that is not already being used by the system. +Please do the following: -**1. Add user:** +**1. Add user to Gitlab instance:** -Add the new user on the Gitlab instance. +Add a new account for the new user on the Gitlab instance. +Note the username and email of the new account. -**2. Setup a new workspace:** +**2. Create User Workspace:** -The above code creates a new workspace for the new user based on _user2_. +Use the [DTaaS CLI](../cli.md) to bring up the workspaces for new users. +This brings up the containers, without the backend authorization. -```bash -cd DTaaS/files -cp -R user2 -cd .. -docker run -d \ - -p :8080 \ - --name "ml-workspace-" \ - -v "${TOP_DIR}/files/:/workspace" \ - -v "${TOP_DIR}/files/:/workspace/common" \ - --env AUTHENTICATE_VIA_JUPYTER="" \ - --env WORKSPACE_BASE_URL="" \ - --shm-size 512m \ - --restart always \ - mltooling/ml-workspace-minimal:0.13.2 -``` - -**3. Add username and password:** +**3. Add backend authorization for the user:** -The following code adds basic authorization for the new user. +- Go to the _docker_ directory -```bash -cd DTaaS/servers/config/gateway -htpasswd auth -``` + ```bash + cd /docker + ``` -**4. Add 'route' for new user:** +- Add three lines to the `conf.server` file -We need to add a new route to the servers ingress. + ```txt + rule.onlyu3.action=auth + rule.onlyu3.rule=PathPrefix(`/alice`) + rule.onlyu3.whitelist = alice@foo.com + ``` -Open the following file with your preffered editor (e.g. VIM/nano). +**4. Restart the docker container responsible for backend authorization.** ```bash -vi DTaaS/servers/config/gateway/dynamic/fileConfig.yml +docker compose -f compose.server.yml --env-file .env up -d --force-recreate traefik-forward-auth ``` -Now add the new route and service for the user. - - -!!! important - foo.com should be replaced with your own domain. - -```yml -http: - routers: - .... - : - entryPoints: - - http - rule: 'Host(`foo.com`) && PathPrefix(`/`)' - middlewares: - - basic-auth - service: - - services: - ... - : - loadBalancer: - servers: - - url: 'http://localhost:' -``` - -**5. Access the new user:** - -Log into the DTaaS application as new user. +**5. The new users are now added to the DTaaS instance, with authorization enabled.** diff --git a/docs/admin/guides/common_workspace_readonly.md b/docs/admin/guides/common_workspace_readonly.md index 6e45c410d..0a2ab72da 100644 --- a/docs/admin/guides/common_workspace_readonly.md +++ b/docs/admin/guides/common_workspace_readonly.md @@ -1,4 +1,4 @@ -# Make common asset area read only +# Make Common Assets Read Only ## Why @@ -8,33 +8,33 @@ In order to make the common area read only, you have to change the install script section performing the creation of user workspaces. -## How - -To make the common assets read-only for user2, -the following changes need to be made to the install script, -which is located one of the following places. +!!! note + These step needs to be performed before installation of the application. -- trial installation: `single-script-install.sh` +## How -- production installation: `DTaas/deploy/install.sh` +To make the common assets read-only for a user, +the following changes need to be made to +the `compose.server.yml` file. -The line `-v "${TOP_DIR}/files/common:/workspace/common:ro"` -was added to make the common workspace read-only for user2. +```docker + ... + user1: + .... + volumes: + - ${DTAAS_DIR}/files/common:/workspace/common:ro + .... -Here's the updated code: + user2: + .... + volumes: + - ${DTAAS_DIR}/files/common:/workspace/common:ro + .... -```sh -docker run -d \ - -p 8091:8080 \ - --name "ml-workspace-user2" \ - -v "${TOP_DIR}/files/user2:/workspace" \ - -v "${TOP_DIR}/files/common:/workspace/common:ro" \ - --env AUTHENTICATE_VIA_JUPYTER="" \ - --env WORKSPACE_BASE_URL="user2" \ - --shm-size 512m \ - --restart always \ - mltooling/ml-workspace-minimal:0.13.2 || true ``` -This ensures that the common area is read-only for user2, -while the user's own (private) assets are still writable. +Please note the `:ro` at the end of the line. This suffix makes +the common assets read only. + +If you want to have the same kind of read only restriction for +new users as well, please make a similar change in `cli/users.server.yml`. diff --git a/docs/admin/guides/hosting_site_without_https.md b/docs/admin/guides/hosting_site_without_https.md index 83fc5a7e8..e89a57716 100644 --- a/docs/admin/guides/hosting_site_without_https.md +++ b/docs/admin/guides/hosting_site_without_https.md @@ -1,6 +1,7 @@ -# Hosting site without https +# Hosting site without HTTPS -In the default trial or production installation setup, -the https connection is provided by the reverse proxy. +It is recommended that the DTaaS application is hosted +behind a secure reverse proxy which provides HTTPS +termination. The DTaaS application by default runs in http mode. So removing the reverse proxy removes the https mode. diff --git a/docs/admin/guides/remove_user.md b/docs/admin/guides/remove_user.md new file mode 100644 index 000000000..5cee0b334 --- /dev/null +++ b/docs/admin/guides/remove_user.md @@ -0,0 +1,47 @@ +# Remove User + +This page provides steps to remove a user from a DTaaS installation. +The username **alice** is used here to illustrate the steps involved in +removing a user account. + +Please do the following: + +**1. Remove an existing user with the [DTaaS CLI](../cli.md).** + +**2. Remove backend authorization for the user:** + +- Go to the _docker_ directory + + ```bash + cd /docker + ``` + +- Remove these three lines from the `conf.server` file + + ```txt + rule.onlyu3.action=auth + rule.onlyu3.rule=PathPrefix(`/alice`) + rule.onlyu3.whitelist = alice@foo.com + ``` + +- Run the command for these changes to take effect: + + ```bash + docker compose -f compose.server.yml --env-file .env up -d --force-recreate traefik-forward-auth + ``` + +The extra users now have no backend authorization. + +**3. Remove users to Gitlab instance (optional):** + +Please see +[gitlab docs](https://docs.gitlab.com/ee/user/profile/account/delete_account.html) +for more help. + +**4. The user account is now deleted.** + +## Caveat + +You cannot delete the two base users that the DTaaS software +is installed with. You can only delete the extra users that +have been added to the software. diff --git a/docs/admin/guides/update_basepath.md b/docs/admin/guides/update_basepath.md deleted file mode 100644 index fc026743f..000000000 --- a/docs/admin/guides/update_basepath.md +++ /dev/null @@ -1,84 +0,0 @@ -# Update basepath/route for the application - - - -:clipboard: - Update the instructions for client docker container. - - -The updates required to make the application -work with basepath (say bar): - -**1. Change the Gitlab OAuth URLs to include basepath:** - -```yml - REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/', - REACT_APP_REDIRECT_URI: 'https://foo.com/bar/Library', - REACT_APP_LOGOUT_REDIRECT_URI: 'https://foo.com/bar', -``` - -**2. Update traefik gateway config (deploy/config/gateway/fileConfig.yml):** - -```yml -http: - routers: - dtaas: - entryPoints: - - http - rule: "Host(`foo.com`)" #remember, there is no basepath for this rule - middlewares: - - basic-auth - service: dtaas - - user1: - entryPoints: - - http - rule: "Host(`foo.com`) && PathPrefix(`/bar/user1`)" - middlewares: - - basic-auth - service: user1 - - # Middleware: Basic authorization - middlewares: - basic-auth: - basicAuth: - usersFile: "/etc/traefik/auth" - removeHeader: true - - services: - dtaas: - loadBalancer: - servers: - - url: "http://localhost:4000" - - user1: - loadBalancer: - servers: - - url: "http://localhost:8090" -``` - -**3. Update deploy/config/client/env.js:** - -See the [client documentation](../client/config.md) for an example. - -**4. Update install scripts:** - -Update _deploy/install.sh_ by adding basepath. For example, add -_WORKSPACE_BASE_URL="bar/"_ for all user workspaces. - -For user1, the docker command changes to: - -```sh -docker run -d \ - -p 8090:8080 \ - --name "ml-workspace-user1" \ - -v "${TOP_DIR}/files/user1:/workspace" \ - -v "${TOP_DIR}/files/common:/workspace/common" \ - --env AUTHENTICATE_VIA_JUPYTER="" \ - --env WORKSPACE_BASE_URL="bar/user1" \ - --shm-size 512m \ - --restart always \ - mltooling/ml-workspace-minimal:0.13.2 || true -``` - -**5. Proceed with install using deploy/install.sh:** diff --git a/docs/admin/host.md b/docs/admin/host.md index c82a9502e..76e086969 100644 --- a/docs/admin/host.md +++ b/docs/admin/host.md @@ -17,19 +17,28 @@ In the new application configuration, there are two OAuth2 applications. The installation requirements to run this docker version of the DTaaS are: -- DNS name for the server -- docker with compose plugin -- User accounts on a gitlab instance -- OAuth2 application registrations +### Docker with Compose Plugin -### Create User Accounts +It is mandatory to have [Docker](https://www.docker.com/) installed +on your computer. We highly recommend using +[Docker Desktop](https://www.docker.com/products/docker-desktop/). -Create user accounts in a gitlab instance for all the users. +### Domain name -The default docker compose file contains two - _user1_ and _user2_. -These names need to be changed to suitable usernames. +The DTaaS software is a web application and is preferably hosted +on a server with a domain name like _foo.com_. +It is also possible to use an IP address in place of domain name. -It is possible to use as well for OAuth2 authorization purposes. +### OAuth Provider + +**[Gitlab Instance](https://about.gitlab.com/install/)** - +The DTaaS uses Gitlab OAuth2.0 authorization for user authorization. +You can either have an on-premise instance of gitlab, or +use [gitlab.com](https://gitlab.com) itself. + +### User Accounts + +Create user accounts in the linked gitlab instance for all the users. ### OAuth2 Application Registration @@ -155,7 +164,7 @@ cp -R files/user1 files/username where _username_ is one of the selected usernames. This command needs to be repeated for all the selected users. -### Configure Authorization Rules for Traefik Forward-Auth +### Configure Authorization Rules for Backend Authorization The Traefik forward-auth microservices requires configuration rules to manage authorization for different URL paths. @@ -221,27 +230,9 @@ through the single page client now. You may have to click Sign in to Gitlab on the Client page and authorize access to the shown application. -## Adding a new user - -To add a new user to your DTaaS instance, follow these steps: - -- Use the [DTaaS CLI](cli.md) to bring up the ML workspaces for new users. - This brings up the containers, without the authorization enforced by Traefik forward-auth. -- Add three lines to the `conf.server` file - - ```txt - rule.onlyu3.action=auth - rule.onlyu3.rule=PathPrefix(`/user3`) - rule.onlyu3.whitelist = user3@emailservice.com - ``` - -Run the appropritate command for a server/local installation: - -```bash -docker compose -f compose.server.yml --env-file .env.server up -d --force-recreate traefik-forward-auth -``` +### Adding a new user -The new users are now added to the DTaaS instance, with authorization enabled. +Please see the [add new user](guides/add_user.md) to add new users. ## References diff --git a/docs/admin/overview.md b/docs/admin/overview.md index bcdda2019..48428da57 100644 --- a/docs/admin/overview.md +++ b/docs/admin/overview.md @@ -4,18 +4,18 @@ The goal is to set up the DTaaS infrastructure in order to enable your users to use the DTaaS. -As an admin you will administrate the users and the servers of the system. ## Optional Requirements There are three optional requirements for installing the DTaaS. -These requirements are not needed for **localhost** installation. +These requirements are not needed for +[**localhost**](./localhost.md) installation. They are only required for installation of the DTaaS on a web server. ### OAuth Provider The DTaaS software is uses OAuth for user authorization. It is -possible to use either _foo.com_ or your own +possible to use either _gitlab.com_ or your own OAuth service provider. ### Domain name @@ -25,12 +25,6 @@ on a server with a domain name like _foo.com_. However, it is possible to install the software on your computer and use access it at _localhost_. -### Reverse Proxy - -The installation setup recommends that the _foo.com_ server is behind a reverse -proxy / load balancer that provides https termination. You can still use -the DTaaS software even if you do not have this reverse proxy. - ## Install The DTaaS can be installed in different ways. Each version serves a different purpose. @@ -38,9 +32,9 @@ Follow the installation that fits your usecase. | Installation Setup | Purpose | |:-----|:-----| -| [Trial installation on localhost](./localhost.md) | Install DTaaS on your computer for a single user; does not need a web server. _This setup also does not require reverse proxy and domain name._ | -| [Production installation on single host](./host.md) | Install DTaaS on server for multiple users. | +| [localhost](./localhost.md) | Install DTaaS on your computer for a single user; does not need a web server. _This setup does not require domain name._ | +| [Server](./host.md) | Install DTaaS on server for multiple users. | | [One vagrant machine](vagrant/single-machine.md) | Install DTaaS on a virtual machine; can be used for single or multiple users. | | [Two vagrant machines](vagrant/two-machines.md) | Install DTaaS on two virtual machines; can be used for single or multiple users. | | | The core DTaaS application is installed on the first virtual machine and all the services (RabbitMQ, MQTT, InfluxDB, Grafana and MongoDB) are installed on second virtual machine. | -| Seperater Packages: [client website](client/docker.md) and [lib microservice](servers/lib/npm.md) | Can be used independently; do not need full installation of DTaaS. | +| [Independent Packages](packages.md) | Can be used independently; do not need full installation of DTaaS. | diff --git a/docs/admin/packages.md b/docs/admin/packages.md new file mode 100644 index 000000000..bf4561d62 --- /dev/null +++ b/docs/admin/packages.md @@ -0,0 +1,27 @@ +# Independent Packages + +The DTaaS development team publishes reusable packages which are then +put together to form the complete DTaaS application. + +The packages are published on +[github](https://github.com/orgs/INTO-CPS-Association/packages?repo_name=DTaaS), +[npmjs](https://www.npmjs.com/org/into-cps-association), and +[docker hub](https://hub.docker.com/u/intocps) repositories. + +The packages on +[github](https://github.com/orgs/INTO-CPS-Association/packages?repo_name=DTaaS) +are published more frequently but are not user tested. +The packages on [npmjs](https://www.npmjs.com/org/into-cps-association) +and [docker hub](https://hub.docker.com/u/intocps) +are published at least once per release. +The regular users are encouraged to use the packages from npm and docker. + +A brief explanation of the packages is given below. + +| Package Name | Description | Documentation for | Availability | +|:----|:----|:----|:----| +| dtaas-web | React web application | [container image](client/docker.md) | [docker hub](https://hub.docker.com/r/intocps/dtaas-web) and [github](https://github.com/INTO-CPS-Association/DTaaS/pkgs/container/dtaas-web) | +| libms |Library microservice | [npm package](servers/lib/npm.md) | [npmjs](https://www.npmjs.com/package/@into-cps-association/libms) and [github](https://github.com/INTO-CPS-Association/DTaaS/pkgs/npm/libms) | +| | | [container image](servers/lib/docker.md) | [docker hub](https://hub.docker.com/r/intocps/libms) and [github](https://github.com/INTO-CPS-Association/DTaaS/pkgs/container/libms) | +| runner | REST API wrapper for multiple scripts/programs | [npm package](../user/servers/execution/runner/README.md) | [npmjs](https://www.npmjs.com/package/@into-cps-association/runner) and [github](https://github.com/INTO-CPS-Association/DTaaS/pkgs/npm/runner) | +| ml-workspace-minimal (fork of [ml-workspace](https://github.com/ml-tooling/ml-workspace)) | User workspace | not available | [docker hub](https://hub.docker.com/r/intocps/ml-workspace-minimal/tags). Please note that this package is **highly experimental** and only v0.15.0-b2 is usable now. | diff --git a/docs/admin/server.png b/docs/admin/server.png index 0eda94395..c7bbaf18f 100644 Binary files a/docs/admin/server.png and b/docs/admin/server.png differ diff --git a/docs/admin/servers/auth.md b/docs/admin/servers/auth.md index 339489f9f..9cbf36905 100644 --- a/docs/admin/servers/auth.md +++ b/docs/admin/servers/auth.md @@ -1,11 +1,5 @@ # OAuth for Traefik Gateway - - -!!! failure - Add screenshots from dtaas-digitaltwin.com demo. - - The traefik gateway is used to serve the DTaaS. All the services provided as part of the application are secured at the traefik gateway. The security is based on [Traefik forward-auth](https://github.com/thomseddon/traefik-forward-auth). @@ -23,7 +17,7 @@ authorization flow. Here are the steps to get started: -**1. Choose Your GitLab Server:** +**1. Choose GitLab Server:** - You need to set up OAuth authorization on a GitLab server. The commercial gitlab.com is not suitable for multi-user authorization @@ -35,52 +29,60 @@ Here are the steps to get started: Select option to generate client secret and also selection option for trusted application. -**2. Determine Your Website's Hostname:** +**2. Determine Website Hostname:** + +Before setting up OAuth on GitLab, decide on the hostname for your website. +It's recommended to use a self-hosted GitLab instance, which you will use in +other parts of the DTaaS application. -- Before setting up OAuth on GitLab, decide on the hostname for your website. - It's recommended to use a self-hosted GitLab instance, which you will use in - other parts of the DTaaS application. +**3. Determine Callback and Logout URLs:** -**3. Define Callback and Logout URLs:** +For the web / server authorization flow to function correctly, +you need two URLs: a _callback URL_ and a _logout URL_. -- For the web / server authorization flow to function correctly, you need two URLs: - a callback URL and a logout URL. - - The callback URL informs the OAuth provider of the - page where - signed-in users should be redirected. It is the landing - homepage of the DTaaS application. +- The callback URL informs the OAuth provider of the + page where + signed-in users should be redirected. It is the landing + homepage of the DTaaS application. (either or ) - - The logout URL is the URL for signout of gitlab and clear authorization +- The logout URL is the URL for signout of gitlab and clear authorization within traefik-forward auth. (either or ). The logout URL is to help users logout of traefik forward-auth. The logout URL should not be entered into Gitlab OAuth application setup. -**4. OAuth Application Creation:** +**4. Create OAuth Application:** + +During the creation of the OAuth application on GitLab, you need to specify +the scope. Choose **_read_user_** scope. + +![Creation of Server OAuth Application](server-oauth.png) + +**5. Copy Application Credentials:** -- During the creation of the OAuth application on GitLab, you need to specify - the scope. Choose _read_user_ scope. +After successfully creating the OAuth application, GitLab generates +an _application ID_ and _client secret_. -**5. Application Credentials:** +Both these values are long string of HEX values that you will need for +your configuration files. -- After successfully creating the OAuth application, GitLab generates - an _application ID_ and _client secret_. - Both these values are long string of HEX values that you will need for - your configuration files. +![Server OAuth Application Credentials](server-oauth2.png) -**6. Required Information from OAuth Application:** +**6. Checklist: Required Information from OAuth Application:** -- You will need the following information from the OAuth application - registered on GitLab: +You will need the following information from +the OAuth application registered on GitLab: -|GitLab Variable Name|Variable Name in .env of docker compose file |Default Value| -|---|---|---| + +| GitLab Variable Name | Variable Name in .env of docker compose file | Default Value | +|:---|:---|:---| |OAuth Provider|OAUTH_URL|[https://gitlab.foo.com/](https://gitlab.foo.com/)| |Application ID|CLIENT_ID| _xx_ | |Application Secret|CLIENT_SECRET| _xx_ | |Callback URL|(to be directly entered in Gitlab OAuth registration)|| |Forward-auth secret|OAUTH_SECRET|_random-secret-string_ (password for forward-auth, can be changed to your preferred string) | |Scopes| read_user || + ## Development Environment @@ -146,7 +148,7 @@ allowing only users mentioned in the whitelist. The rules in _conf.*_ file are not dynamically loaded into the **traefik-forward-auth** microservice. Any change in the _conf_ file requires -retart of **traefik-forward-auth** for the changes to take effect. +restart of **traefik-forward-auth** for the changes to take effect. All the existing user sessions get invalidated when the **traefik-forward-auth** restarts. diff --git a/docs/admin/servers/lib/docker.md b/docs/admin/servers/lib/docker.md index 3fa89dcec..73168ccaa 100644 --- a/docs/admin/servers/lib/docker.md +++ b/docs/admin/servers/lib/docker.md @@ -1,5 +1,20 @@ # Host Library Microservice + + +!!! failure + update the page + + + + +!!! warning + The default setting in docker compose file exposes + all user files at . + The `compose.server.yml` file needs to updated to + expose another directory like common assets directory. + + The **lib microservice** is a simplified file manager providing graphQL API. It has three features: @@ -24,87 +39,8 @@ There is a skeleton file structure in [DTaaS codebase](https://github.com/INTO-CPS-Association/DTaaS/tree/feature/distributed-demo/files). You can copy and create file system for your users. -## :arrow_down: Install - -The package is available in Github -[packages registry](https://github.com/orgs/INTO-CPS-Association/packages) -and on -[npmjs](https://www.npmjs.com/package/@into-cps-association/libms). - -Set the registry and install the package with the one of -the two following commands - -```bash -sudo npm install -g @into-cps-association/libms # requires no login -sudo npm config set @into-cps-association:registry https://npm.pkg.github.com -``` - -The _github package registry_ asks for username and password. The username is -your Github username and the password is your Github -[personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens). -In order for the npm to download the package, your personal access token -needs to have _read:packages_ scope. - -## :gear: Configure - -The microservices requires config specified in INI format. -The template configuration file is: - -```ini -PORT='4001' -MODE='local' -LOCAL_PATH='/Users//DTaaS/files' -LOG_LEVEL='debug' -APOLLO_PATH='/lib' or '' -GRAPHQL_PLAYGROUND='false' or 'true' -``` - -The `LOCAL_PATH` variable is the absolute filepath to the -location of the local directory which will be served to users -by the Library microservice. - -Replace the default values the appropriate values for your setup. - ## :rocket: Use -Display help. - -```bash -libms -h -``` - -The config is saved `.env` file by convention. The **libms** looks for -`.env` file in the working directory from which it is run. -If you want to run **libms** without explicitly specifying the configuration -file, run - -```bash -libms -``` - -To run **libms** with a custom config file, - -```bash -libms -c FILE-PATH -libms --config FILE-PATH -``` - -If the environment file is named something other than `.env`, -for example as `.env.development`, you can run - -```sh -libms -c ".env.development" -``` - -You can press `Ctl+C` to halt the application. -If you wish to run the microservice in the background, use - -```bash -nohup libms [-c FILE-PATH] & disown -``` - -The lib microservice is now running and ready to serve files, functions, and models. - ## Service Endpoint The URL endpoint for this microservice is located at: `localhost:PORT/lib` diff --git a/docs/admin/servers/lib/npm.md b/docs/admin/servers/lib/npm.md index 3fa89dcec..9c2f91d0e 100644 --- a/docs/admin/servers/lib/npm.md +++ b/docs/admin/servers/lib/npm.md @@ -1,5 +1,11 @@ # Host Library Microservice + + +!!! failure + update the page + + The **lib microservice** is a simplified file manager providing graphQL API. It has three features: diff --git a/docs/admin/servers/server-oauth.png b/docs/admin/servers/server-oauth.png new file mode 100644 index 000000000..7b72ba50f Binary files /dev/null and b/docs/admin/servers/server-oauth.png differ diff --git a/docs/admin/servers/server-oauth2.png b/docs/admin/servers/server-oauth2.png new file mode 100644 index 000000000..ce3677584 Binary files /dev/null and b/docs/admin/servers/server-oauth2.png differ diff --git a/docs/bugs.md b/docs/bugs.md index 9dfb24b3b..59af3ab7e 100644 --- a/docs/bugs.md +++ b/docs/bugs.md @@ -7,7 +7,7 @@ ### ML Workspace -- the docker container slows down a bit after ten days. +- the docker container might down a bit after two weeks. The only known solution is to restart the docker container. You don't need to restart the complete DTaaS platform, restart of the docker container of ml-workspace is sufficient. diff --git a/docs/developer/docker/docker.md b/docs/developer/docker/docker.md new file mode 100644 index 000000000..7c48577c4 --- /dev/null +++ b/docs/developer/docker/docker.md @@ -0,0 +1,135 @@ +# Docker workflow for DTaaS + +This readme will explain the building and use of different docker files +for use in development and installation of the DTaaS software. + +**NOTE**: A local docker installation with compose plugin is a pre-requisite +for using docker workflows. + +## Folder Structure + +There are two dockerfiles for building the containers: + +- **client.dockerfile**: Dockerfile for building + the client application container. +- **libms.dockerfile**: Dockerfile for building the library + microservice container from source code. +- **libms.npm.dockerfile**: Dockerfile for building the library + microservice container from published npm package at npmjs.com. + This Dockerfile is only used during publishing. It is used neither + in the development builds nor in Github actions. + +There is a specific compose file for development: + +The **compose.dev.yml:** file is the docker compose file +for development environment. + +## Build and Publish Docker Images + +### Project Managers + +Build and publish the docker images. This step is required only for +the publication of images to Docker Hub. + +:stop_sign: This publishing step is managed +only by project managers. Regular developers can skip this step. + +#### React Website + +```sh +docker build -t intocps/dtaas-web:latest -f ./docker/client.dockerfile . +docker tag intocps/dtaas-web:latest intocps/dtaas-web: +docker push intocps/dtaas-web:latest +docker push intocps/dtaas-web: +``` + +To tag version **0.3.1** for example, use + +```sh +docker tag intocps/dtaas-web:latest intocps/dtaas-web:0.3.1 +``` + +To test the react website container on localhost, please use + +```bash +docker run -d \ + -v ${PWD}/client/config/local.js:/dtaas/client/build/env.js \ + -p 4000:4000 intocps/dtaas-web:latest +``` + +#### Library Microservice + +The Dockerfile of library microservice has `VERSION` argument. +This argument helps pick the right package version from . + +```sh +docker login -u -p +docker build -t intocps/libms:latest -f ./docker/libms.npm.dockerfile . +docker push intocps/libms:latest +docker build --build-arg="VERSION=" \ + -t intocps/libms: -f ./docker/libms.npm.dockerfile . +docker push intocps/libms: +``` + +To tag version 0.3.1 for example, use + +```sh +docker build --build-arg="VERSION=0.3.1" \ + -t intocps/libms:0.3.1 -f ./docker/libms.npm.dockerfile . +``` + +To test the library microservice on localhost, please use + +```bash +docker run -d -v ${PWD}/files:/dtaas/libms/files \ + -p 4001:4001 intocps/libms:latest +``` + +### Developers + +Use of docker images is handy for developers as well. It is suggested +that developers build the required images locally on their computer and +use them for development purposes. The images can be built using + +```sh +docker compose -f compose.dev.yml build +``` + +## Running Docker Containers + +Follow these steps to use the application with docker. + +The DTaaS application requires multiple configuration files. The list of +configuration files to be modified are given for each scenario. + +### Development Environment + +This scenario is for software developers. + +The configuration files to be updated are: + +1. **docker/.env** : + please see [docker installation docs](../../admin/host.md) for help + with updating this config file +1. **docker/conf.dev** : + please see [docker installation docs](../../admin/host.md) for help + with updating this config file +1. **client/config/local.js** + please see [client config](../../admin/client/config.md) for help + with updating this config file +1. **servers/lib/config/.env.default** + please see [lib config](../../admin/servers/lib/docker.md) for help + with updating this config file + +The docker commands need to be executed from this directory (`docker`). +The relevant docker commands are: + +```bash +docker compose -f compose.dev.yml up -d #start the application +docker compose -f compose.dev.yml down #terminate the application +``` + +### Access the Application + +You should access the application through the PORT mapped to the Traefik container. +e.g. `localhost` diff --git a/docs/developer/docker/traefik-oauth.png b/docs/developer/docker/traefik-oauth.png new file mode 100644 index 000000000..95a7ae391 Binary files /dev/null and b/docs/developer/docker/traefik-oauth.png differ diff --git a/docs/developer/oauth.md b/docs/developer/oauth.md index 43c64ec9c..7e285e10c 100644 --- a/docs/developer/oauth.md +++ b/docs/developer/oauth.md @@ -22,3 +22,20 @@ The installation requirements to run this docker version of the DTaaS are: for trusted application. - DNS name (optional, required only when the DTaaS is to be - deployed on a web server) + +## React Website Development Environment + +There needs to be a valid callback and logout URLs for development and testing +purposes. You can use the same oauth application id for both development, testing +and deployment scenarios. Only the callback and logout URLs change. It is possible +to register multiple callback URLs in one oauth application. In order to use oauth +for development and testing on developer computer (localhost), you need to add the +following to oauth callback URL. + +```txt +DTaaS application URL: http://localhost:4000 +Callback URL: http://localhost:4000/Library +Logout URL: http://localhost:4000 +``` + +The port 4000 is the default port for running the client website. diff --git a/docs/developer/system/DTaaS.drawio b/docs/developer/system/DTaaS.drawio index cea224c00..2d8230ecf 100755 --- a/docs/developer/system/DTaaS.drawio +++ b/docs/developer/system/DTaaS.drawio @@ -1,6 +1,6 @@ - + - + @@ -2923,7 +2923,7 @@ - + @@ -3565,89 +3565,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -3734,63 +3651,63 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -3798,7 +3715,7 @@ - + @@ -3807,16 +3724,16 @@ - + - + - + - + @@ -3872,13 +3789,114 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/thanks.md b/docs/thanks.md index 0ca23f27d..aa56ca729 100644 --- a/docs/thanks.md +++ b/docs/thanks.md @@ -8,6 +8,14 @@ Cláudio Ângelo Gonçalves Gomes, Dmitri Tcherniak, Elif Ecem Bas, Giuseppe Abbiati, Hao Feng, Henrik Ejersbo, Tanusree Roy, Farshid Naseri +## Example Contributors + + + +!!! failure + Cite all example providers and recent users + + ## Documentation 1. Talasila, P., Gomes, C., Mikkelsen, P. H., Arboleda, S. G., Kamburjan, diff --git a/docs/user/examples/incubator-NuRV-monitor-validation/README.md b/docs/user/examples/incubator-NuRV-monitor-validation/README.md new file mode 100644 index 000000000..bdc74bc9d --- /dev/null +++ b/docs/user/examples/incubator-NuRV-monitor-validation/README.md @@ -0,0 +1,144 @@ +# Incubator Digital Twin validation with NuRV Monitor + +## Overview + +This example demonstrates how to validate some digital twin components using +FMU monitors (in this example, the monitors are generated with NuRV[1]). + +## Simulated scenario + +This example validates some components of the [Incubator digital twin](../incubator/README.md), +by performing a simulation in which the commponents are wrapped inside FMUs, +and are then inspected at runtime by some a FMU monitor generated by NuRV. + +The input data for the simulation is generated by a purpose-built +FMU component named _source_, which supplies testing data to the _anomaly detector_, +simulating an anomaly occurring at time t=60s. An additional component, +_watcher_, is employed to verify whether the _energy saver_ activates in response +to an anomaly reported by the _anomaly detector_. + +The output of the watcher is the passed to the monitor, which ensures that when +an anomaly is detected, the _energy saver_ activates within a maximum of +three simulation cycles. + +## Example structure + +A diagram depicting the logical software structure of the example can be seen below. + +![DT structure](dt-structure.png) + +## Digital Twin configuration + +The example uses the following assets: + +| Asset Type | Names of Assets | Visibility | Reuse in other Examples | +|:---|:---|:---|:---| +| Models | anomaly_detection.fmu | Private | No | +| | energy_saver.fmu | Private | No | +| | Source.fmu | Private | No | +| | Watcher.fmu | Private | No | +| Specification | safe-operation.smv | Private | No | +| Tool | maestro-2.3.0-jar-with-dependencies.jar | Common | Yes | + +The _safe-operation.smv_ file contains the default monitored specification as +described in the [Simulated scenario section](#simulated-scenario). + +## Lifecycle phases + +The lifecycle phases for this example include: + +| Lifecycle Phase | Completed Tasks | +| -------- | ------- | +| Create | Installs Java Development Kit for Maestro tool
Generates and compiles the monitor FMU | +| Execute | Produces and stores output in data/incubator-NuRV-monitor-validation/output directory| +| Clean | Clears run logs and outputs | + +If required, change the execute permissions of lifecycle scripts you need to execute. +This can be done using the following command + +```bash +chmod +x lifecycle/{script} +``` + +where {script} is the name of the script, e.g. _create_, _execute_ etc. + +## Run the example + +To run the example, change your present directory. + +```bash +cd /workspace/examples/digital_twins/incubator-NuRV-monitor-validation +``` + +If required, change the execute permission of lifecycle scripts +you need to execute, for example: + +```bash +chmod +x lifecycle/create +``` + +Now, run the following scripts: + +### Create + +- Installs Open Java Development Kit 17 in the workspace. +- Generates and compiles the monitor FMU from the NuRV specification + +```bash +lifecycle/create +``` + +### Execute + +Run the the Digital Twin. Since this is a co-simulation based +digital twin, the Maestro co-simulation tool executes co-simulation +using the FMU models. + +```bash +lifecycle/execute +``` + +### Analyze phase + +Process the output of co-simulation to produce a plot at: +`/workspace/examples/data/incubator-NuRV-monitor-validation/output/plots`. + +```bash +lifecycle/analyze +``` + +A sample plot is given here. + +![output plot](incubator-NuRV-monitor-validation_plot.png) + +In the plot, four color-coded indicators provide a visual representation of +distinct values. The blue line depicts the simulated temperature, while orange +one represents the temperature estimate. A red dashed line indicates the +target temperature set by the energy saver component. The green line shows +the monitor's output verdict. As observed, when there is a disparity +between the estimated and actual temperatures, the energy saver adjusts +the target temperature downward, indicating that the component is working +properly. Thus, the output of the monitor is fixed at `unknown` (`0`), +signifying that the monitoring property is not violated. + +#### Examine the results + +The results can be found in the +_/workspace/examples/data/incubator-NuRV-monitor-validation/output_ directory +where the logs are also included. + +Figures of the output results can be found in the +_/workspace/examples/data/incubator-NuRV-monitor-validation/output_ +directory. + +### Terminate phase + +Terminate to clean up the debug files and co-simulation output files. + +```bash +lifecycle/terminate +``` + +## References + +1. More information about NuRV is available [here](https://es-static.fbk.eu/tools/nurv/). diff --git a/docs/user/examples/incubator-NuRV-monitor-validation/dt-structure.png b/docs/user/examples/incubator-NuRV-monitor-validation/dt-structure.png new file mode 100644 index 000000000..47ba9be57 Binary files /dev/null and b/docs/user/examples/incubator-NuRV-monitor-validation/dt-structure.png differ diff --git a/docs/user/examples/incubator-NuRV-monitor-validation/incubator-NuRV-monitor-validation_plot.png b/docs/user/examples/incubator-NuRV-monitor-validation/incubator-NuRV-monitor-validation_plot.png new file mode 100644 index 000000000..c1f8b6e67 Binary files /dev/null and b/docs/user/examples/incubator-NuRV-monitor-validation/incubator-NuRV-monitor-validation_plot.png differ diff --git a/docs/user/examples/index.md b/docs/user/examples/index.md index 60bed3357..644c19b09 100644 --- a/docs/user/examples/index.md +++ b/docs/user/examples/index.md @@ -33,9 +33,12 @@ to use the examples in the following order. 1. [Desktop Robotti and RabbitMQ](./drobotti-rmqfmu/README.md) 1. [Water Treatment Plant and OPC-UA](./opc-ua-waterplant/README.md) 1. [Three Water Tanks with DT Manager Framework](./three-tank/README.md) -1. [Flex Cell Digital Twin with Two Industrial Robots](./flex-cell/README.md) +1. [Flex Cell with Two Industrial Robots](./flex-cell/README.md) 1. [Incubator](./incubator/README.md) 1. [Firefighters in Emergency Environments](./o5g/README.md) -1. [Incubator with NuRV run-time monitor service](./incubator-NuRV-monitor-service/README.md) +1. [Mass Spring Damper with NuRV Runtime Monitor](./mass-spring-damper-monitor/README.md) +1. [Incubator with NuRV Runtime Monitor](./incubator-NuRV-monitor-validation/README.md) +1. [Incubator with NuRV Runtime Monitor Service](./incubator-NuRV-monitor-service/README.md) +1. [Water Tank Fault Injection with NuRV Runtime Monitor](./water_tank_FI_monitor/README.md) :material-download: [DTaaS examples](https://github.com/INTO-CPS-Association/DTaaS-examples) diff --git a/docs/user/examples/mass-spring-damper-monitor/README.md b/docs/user/examples/mass-spring-damper-monitor/README.md new file mode 100644 index 000000000..804b58146 --- /dev/null +++ b/docs/user/examples/mass-spring-damper-monitor/README.md @@ -0,0 +1,139 @@ +# Mass Spring Damper with NuRV Runtime Monitor + +## Overview + +This digital twin is derives from te Mass String Damper digital twin. + +The mass spring damper digital twin (DT) comprises two mass spring dampers +and demonstrates how a co-simulation based DT can be used within DTaaS. +This version of the example is expanded with a monitor generated by NuRV. +More information about NuRV is available [here](https://es-static.fbk.eu/tools/nurv/). + +## Example Diagram + +![Mass Spring Damper System](mass-spring-damper_multibody_system.png) + +## Example Structure + +There are two simulators included in the study, each representing a +mass spring damper system. The first simulator calculates the mass +displacement and speed of $m_1$ for a given force $F_k$ acting on mass $m_1$. +The second simulator calculates force $F_k$ given a displacement and speed of +mass $m_1$. By coupling these simulators, the evolution of the position of +the two masses is computed. + +Additionally, a monitor is inserted in the simulation to check at runtime +whether the displacement of the two masses stays below a maximum threshold. + +![Mass Spring Damper Structure](dt-structure.png) + +## Digital Twin Configuration + +This example uses two models and one tool. The specific assets used are: + +| Asset Type | Names of Assets | Visibility | Reuse in Other Examples | +|:---|:---|:---|:---| +| Models | MassSpringDamper1.fmu | Private | Yes | +| | MassSpringDamper2.fmu | Private | Yes | +| | m2.fmu | Private | No | +| | RtI.fmu | Private | Yes | +| Specification | m2.smv | Private | No | +| Tool | maestro-2.3.0-jar-with-dependencies.jar | Common | Yes | + +The `co-sim.json` and `time.json` +are two DT configuration files used for executing the digital twin. +You can change these two files to customize the DT to your needs. + +## Lifecycle Phases + +| Lifecycle Phase | Completed Tasks | +| -------- | ------- | +| Create | Installs Java Development Kit for Maestro tool
Generates and compiles the monitor FMU | +| Execute | Produces and stores output in data/mass-spring-damper-monitor/output directory| +| Clean | Clears run logs and outputs | + +## Run the example + +To run the example, change your present directory. + +```bash +cd /workspace/examples/digital_twins/mass-spring-damper-monitor +``` + +If required, change the execute permission of lifecycle scripts +you need to execute, for example: + +```bash +chmod +x lifecycle/create +``` + +Now, run the following scripts: + +### Create + +- Installs Open Java Development Kit 17 in the workspace. +- Generates and compiles the monitor FMU from the NuRV specification + +```bash +lifecycle/create +``` + +### Execute + +Run the the Digital Twin. Since this is a co-simulation based +digital twin, the Maestro co-simulation tool executes co-simulation +using the two FMU models. + +```bash +lifecycle/execute +``` + +### Analyze phase + +Process the output of co-simulation to produce a plot at: +`/workspace/examples/data/mass-spring-damper-monitor/output/plots`. + +```bash +lifecycle/analyze +``` + +A sample plot is given here. + +![sample plot](mass-spring-damper-monitor_plot.png) + +In the plot, three color-coded indicators are used to represent different +values. The blue line shows the distance between the two masses, while the +green indicates the monitor's verdict. A red dashed +line serves as a reference point, marking the distance checked by the +monitor. Since the distance of the masses is always below the threshold, the +output of the monitor is fixed to `unknown` (`0`). + +#### Examine the results + +The results can be found in the +_/workspace/examples/data/mass-spring-damper-monitor/output directory_. + +You can also view run logs in the +_/workspace/examples/digital_twins/mass-spring-damper-monitor_. + +### Terminate phase + +Terminate to clean up the debug files and co-simulation output files. + +```bash +lifecycle/terminate +``` + +## References + +More information about co-simulation techniques and mass spring damper +case study are available in: + +```txt +Gomes, Cláudio, et al. "Co-simulation: State of the art." +arXiv preprint arXiv:1702.00686 (2017). +``` + +The source code for the models used in this DT are available in +[mass spring damper](https://github.com/INTO-CPS-Association/example-mass_spring_damper) +github repository. diff --git a/docs/user/examples/mass-spring-damper-monitor/dt-structure.png b/docs/user/examples/mass-spring-damper-monitor/dt-structure.png new file mode 100644 index 000000000..c06427927 Binary files /dev/null and b/docs/user/examples/mass-spring-damper-monitor/dt-structure.png differ diff --git a/docs/user/examples/mass-spring-damper-monitor/mass-spring-damper-monitor_plot.png b/docs/user/examples/mass-spring-damper-monitor/mass-spring-damper-monitor_plot.png new file mode 100644 index 000000000..7800a3a09 Binary files /dev/null and b/docs/user/examples/mass-spring-damper-monitor/mass-spring-damper-monitor_plot.png differ diff --git a/docs/user/examples/mass-spring-damper-monitor/mass-spring-damper_multibody_system.png b/docs/user/examples/mass-spring-damper-monitor/mass-spring-damper_multibody_system.png new file mode 100644 index 000000000..de2c5699b Binary files /dev/null and b/docs/user/examples/mass-spring-damper-monitor/mass-spring-damper_multibody_system.png differ diff --git a/docs/user/examples/water_tank_FI_monitor/README.md b/docs/user/examples/water_tank_FI_monitor/README.md new file mode 100644 index 000000000..dcc7d7f1c --- /dev/null +++ b/docs/user/examples/water_tank_FI_monitor/README.md @@ -0,0 +1,149 @@ +# Water Tank Fault Injection with NuRV Runtime Monitor + +## Overview + +This example shows a fault injection (FI) enabled digital twin (DT). +A live DT is subjected to simulated faults received from the environment. +The simulated faults is specified as part of DT configuration and can be +changed for new instances of DTs. +This version of the example is expanded with a monitor generated by NuRV. +More information about NuRV is available [here](https://es-static.fbk.eu/tools/nurv/). + +In this co-simulation based DT, a watertank case-study is used; co-simulation +consists of a tank and controller. The goal of which is to keep +the level of water in the tank between ```Level-1``` and ```Level-2```. +The faults are injected into output of the water tank +controller (__Watertankcontroller-c.fmu__) +from 12 to 20 time units, such that +the tank output is closed for a period of time, leading to the water level +increasing in the tank beyond the desired level (```Level-2```). +Additionally, a monitor is inserted in the simulation to check at runtime +whether the level of the water stays below a maximum threshold. + +## Example Diagram + +![Water Tank System](watertank.png) + +## Example Structure + +![Water Tank Structure](dt-structure.png) + +## Digital Twin Configuration + +This example uses two models and one tool. +The specific assets used are: + +| Asset Type | Names of Assets | Visibility | Reuse in Other Examples | +|:---|:---|:---|:---| +| Models | watertankcontroller-c.fmu | Private | Yes | +| | singlewatertank-20sim.fmu | Private | Yes | +| | m1.fmu | Private | No | +| | RtI.fmu | Private | Yes | +| Specification | m1.smv | Private | No | +| Tool | maestro-2.3.0-jar-with-dependencies.jar | Common | Yes | + +The `multimodelFI.json` and `simulation-config.json` +are two DT configuration files used for executing the digital twin. +You can change these two files to customize the DT to your needs. + +:fontawesome-solid-circle-info: The faults are defined in __wt_fault.xml__. + +## Lifecycle Phases + +| Lifecycle Phase | Completed Tasks | +| -------- | ------- | +| Create | Installs Java Development Kit for Maestro tool
Generates and compiles the monitor FMU | +| Execute | Produces and stores output in data/water_tank_FI_monitor/output directory| +| Clean | Clears run logs and outputs | + +## Run the example + +To run the example, change your present directory. + +```bash +cd /workspace/examples/digital_twins/water_tank_FI_monitor +``` + +If required, change the execute permission of lifecycle scripts +you need to execute, for example: + +```bash +chmod +x lifecycle/create +``` + +Now, run the following scripts: + +### Create + +Installs Open Java Development Kit 17 and pip dependencies. +The pandas and matplotlib are the pip dependencies installated. +The monitor FMU from the NuRV specification is generated and compiled. + +```bash +lifecycle/create +``` + +### Execute + +Run the co-simulation. Generates the co-simulation output.csv file +at `/workspace/examples/data/water_tank_FI_monitor/output`. + +```bash +lifecycle/execute +``` + +### Analyze phase + +Process the output of co-simulation to produce a plot at: +`/workspace/examples/data/water_tank_FI_monitor/output/plots/`. + +```bash +lifecycle/analyze +``` + +A sample plot is given here. + +![sample plot](water_tank_FI_monitor_plot.png) + +In the plot, four color-coded indicators are used to represent different +values. The blue line shows the water tank level, while orange represents +the control output and green indicates the monitor's verdict. A red dashed +line serves as a reference point, marking the level checked by the +monitor. As the water level exceeds this threshold, the monitor's verdict +changes from `unknown` (`0`) to `false` (`2`). + +#### Examine the results + +The results can be found in the +_/workspace/examples/data/water_tank_FI_monitor/output directory_. + +You can also view run logs in the +_/workspace/examples/digital_twins/water_tank_FI_monitor_. + +### Terminate phase + +Clean up the temporary files and delete output plot + +```bash +lifecycle/terminate +``` + +## References + +More details on this case-study can be found in the paper: + +```txt +M. Frasheri, C. Thule, H. D. Macedo, K. Lausdahl, P. G. Larsen and +L. Esterle, "Fault Injecting Co-simulations for Safety," +2021 5th International Conference on System Reliability and Safety (ICSRS), +Palermo, Italy, 2021. +``` + +The fault-injection plugin is an extension to the Maestro co-orchestration +engine that enables injecting inputs and outputs of FMUs in an FMI-based +co-simulation with tampered values. +More details on the plugin can be found in +[fault injection](https://github.com/INTO-CPS-Association/fault-injection-maestro) +git repository. The source code for this example is also in the same github +repository in a +[example directory](https://github.com/INTO-CPS-Association/fault-injection-maestro/blob/development/fi_example/README.md). diff --git a/docs/user/examples/water_tank_FI_monitor/dt-structure.png b/docs/user/examples/water_tank_FI_monitor/dt-structure.png new file mode 100644 index 000000000..5f9babd7e Binary files /dev/null and b/docs/user/examples/water_tank_FI_monitor/dt-structure.png differ diff --git a/docs/user/examples/water_tank_FI_monitor/water_tank_FI_monitor_plot.png b/docs/user/examples/water_tank_FI_monitor/water_tank_FI_monitor_plot.png new file mode 100644 index 000000000..b116ae80b Binary files /dev/null and b/docs/user/examples/water_tank_FI_monitor/water_tank_FI_monitor_plot.png differ diff --git a/docs/user/examples/water_tank_FI_monitor/watertank.png b/docs/user/examples/water_tank_FI_monitor/watertank.png new file mode 100644 index 000000000..363a5ab8b Binary files /dev/null and b/docs/user/examples/water_tank_FI_monitor/watertank.png differ diff --git a/docs/user/servers/lib/LIB-MS.md b/docs/user/servers/lib/LIB-MS.md index 14149bd0a..2df0292ac 100644 --- a/docs/user/servers/lib/LIB-MS.md +++ b/docs/user/servers/lib/LIB-MS.md @@ -14,15 +14,40 @@ This document provides instructions for using the library microservice. Please see [assets](assets.md) for a suggested storage conventions of your library assets. -Once the assets are stored in the library, -you can access the server's endpoint by typing -in the following URL: `http://foo.com/lib`. +Once the assets are stored in the library, they become +available in user workspace. -The URL opens a graphql playground. -You can check the query schema and try sample queries here. -You can also send graphql queries as HTTP POST requests and get responses. +## Application Programming Interface (API) + +The lib microservice application provides services at +two end points: + +**GraphQL API Endpoint:** `http://foo.com/lib`. + +**HTTP Endpoint:** `http://foo.com/lib` +(need to be [enabled](../../../admin/servers/lib/npm.md)). + +### HTTP protocol + +Endpoint: `localhost:PORT/lib/files` + +This option needs to be enabled with `-H http.json` flag. +The regular file upload and download options become available. -## API Queries +Here are sample screenshots. + +![top-level](top-level.png) + +![inside](inside.png) + +### GraphQL protocol + +Endpoint: `localhost:PORT/lib` + +The `http://foo.com/lib` URL opens a graphql playground. + +You can check the query schema and try sample queries here. +The graphql queries need to be sent as HTTP POST requests and get responses. The library microservice services two API calls: diff --git a/docs/user/servers/lib/inside.png b/docs/user/servers/lib/inside.png new file mode 100644 index 000000000..9b6b6042f Binary files /dev/null and b/docs/user/servers/lib/inside.png differ diff --git a/docs/user/servers/lib/top-level.png b/docs/user/servers/lib/top-level.png new file mode 100644 index 000000000..2947251df Binary files /dev/null and b/docs/user/servers/lib/top-level.png differ diff --git a/docs/user/website/gitlab-auth-1.png b/docs/user/website/gitlab-auth-1.png new file mode 100644 index 000000000..62187b66e Binary files /dev/null and b/docs/user/website/gitlab-auth-1.png differ diff --git a/docs/user/website/gitlab-auth-2.png b/docs/user/website/gitlab-auth-2.png new file mode 100644 index 000000000..135daa638 Binary files /dev/null and b/docs/user/website/gitlab-auth-2.png differ diff --git a/docs/user/website/gitlab-sso-3.png b/docs/user/website/gitlab-sso-3.png index e44dca11a..3a8d234cb 100755 Binary files a/docs/user/website/gitlab-sso-3.png and b/docs/user/website/gitlab-sso-3.png differ diff --git a/docs/user/website/index.md b/docs/user/website/index.md index 9ffbf4076..062861aa5 100644 --- a/docs/user/website/index.md +++ b/docs/user/website/index.md @@ -3,38 +3,61 @@ This page contains a screenshot driven preview of the website serving the DTaaS software platform. -## Login to enter the DTaaS software platform +## Visit the DTaaS Application -![Login](login.png) +Start off by simply visiting the website +of the DTaaS instance for which you are a user. -The screen presents with HTTP authorization form. You can enter the user -credentials. If the DTaaS is being served over HTTPS secure communication -protocol, the username and password are secure. +![Visit the URL](visit-dtaas-1.png) -## Start the Authorization +## Redirected to Authorization Provider -You are now logged into the DTaaS server. The DTaaS uses third-party authorization +You will be redirected to the Gitlab +Authorization for DTaaS. + +![Gitlab Sign on](gitlab-auth-1.png) + +Enter your email/username and password. +If the email ID registered with DTaaS, is +the same as your Google Login email ID, you +can also opt to sign in using Google. + +You will be redirected to the OAuth +Application page. + +## Permit DTaaS Server to Use Gitlab + +![Server Auth Permission](gitlab-auth-2.png) + +Click on Authorize to allow the OAuth +application to access the information +connected to your Gitlab account. This +is a necessary step. + +You are now logged into the DTaaS server. +You will be redirected to the login page +of the DTaaS website. + +The DTaaS website uses +an additional layer of security - +the third-party authorization protocol known as -[OAuth](https://auth0.com/docs/get-started/authorization-and-authorization-flow/authorization-code-flow-with-proof-key-for-code-exchange-pkce). +[OAuth](https://auth0.com/docs/get-started/authentication-and-authorization-flow/authorization-code-flow-with-pkce). This protocol provides secure access to a DTaaS installation if users have a working active accounts at the selected OAuth -service provider. The DTaaS uses Gitlab as OAuth provider. +service provider. This also uses Gitlab as OAuth provider. ![Gitlab signin button](gitlab-sso-1.png) You can see the Gitlab signin button. A click on this button takes you to -Gitlab instance providing authorization for DTaaS. - -## Authorize at Gitlab - -The username and password authorization takes place on the gitlab website. -Enter your username and password in the login form. - -![Gitlab login form](gitlab-sso-2.png) +Gitlab instance providing authorization for DTaaS. You will not need to sign in to +Gitlab again, unless you have explicitly +logged out +of your Gitlab account. -## Permit DTaaS to Use Gitlab +## Permit DTaaS Website to Use Gitlab -The DTaaS application needs your permission to use your Gitlab account for +The DTaaS website needs your permission to use your Gitlab account for authorization. Click on **Authorize** button. ![Gitlab OAuth permission](gitlab-sso-3.png) diff --git a/docs/user/website/visit-dtaas-1.png b/docs/user/website/visit-dtaas-1.png new file mode 100644 index 000000000..2a9b234a8 Binary files /dev/null and b/docs/user/website/visit-dtaas-1.png differ diff --git a/mkdocs-github.yml b/mkdocs-github.yml index c6d89b284..58d36cede 100644 --- a/mkdocs-github.yml +++ b/mkdocs-github.yml @@ -29,22 +29,16 @@ nav: - Base Box: admin/vagrant/base-box.md - Single Machine: admin/vagrant/single-machine.md - Two Machines: admin/vagrant/two-machines.md - - Separate Packages: - - Website: - - config: admin/client/config.md - - docker: admin/client/docker.md - - Library microservice: - - npm: admin/servers/lib/npm.md - - docker: admin/servers/lib/docker.md + - Independent Packages: admin/packages.md - Services: admin/services.md - Guides: - - New user: admin/guides/add_user.md - - New Service: admin/guides/add_service.md + - Add user: admin/guides/add_user.md + - Remove user: admin/guides/remove_user.md + - Add Service: admin/guides/add_service.md - Install Gitlab: admin/guides/gitlab.md - Link services: admin/guides/link_service.md - Common Assets: admin/guides/common_workspace_readonly.md - HTTP Mode: admin/guides/hosting_site_without_https.md - - Basepath: admin/guides/update_basepath.md - User: - Motivation: user/motivation.md - Features: user/features.md @@ -67,7 +61,10 @@ nav: - Flex Cell: user/examples/flex-cell/README.md - Incubator: user/examples/incubator/README.md - Firefighters: user/examples/o5g/README.md + - Mass Spring Damper with NuRV: user/examples/mass-spring-damper-monitor/README.md + - Incubator with NuRV: user/examples/incubator-NuRV-monitor-validation/README.md - Incubator with NuRV Service: user/examples/incubator-NuRV-monitor-service/README.md + - Water Tank Fault Injection with NuRV: user/examples/water_tank_FI_monitor/README.md - Codebase: https://github.com/INTO-CPS-Association/DTaaS-examples - FAQ: FAQ.md - Developer: diff --git a/mkdocs.yml b/mkdocs.yml index 336bd2d23..083354cc6 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -29,22 +29,16 @@ nav: - Base Box: admin/vagrant/base-box.md - Single Machine: admin/vagrant/single-machine.md - Two Machines: admin/vagrant/two-machines.md - - Separate Packages: - - Website: - - config: admin/client/config.md - - docker: admin/client/docker.md - - Library microservice: - - npm: admin/servers/lib/npm.md - - docker: admin/servers/lib/docker.md + - Independent Packages: admin/packages.md - Services: admin/services.md - Guides: - - New user: admin/guides/add_user.md - - New Service: admin/guides/add_service.md + - Add user: admin/guides/add_user.md + - Remove user: admin/guides/remove_user.md + - Add Service: admin/guides/add_service.md - Install Gitlab: admin/guides/gitlab.md - Link services: admin/guides/link_service.md - Common Assets: admin/guides/common_workspace_readonly.md - HTTP Mode: admin/guides/hosting_site_without_https.md - - Basepath: admin/guides/update_basepath.md - User: - Motivation: user/motivation.md - Features: user/features.md @@ -67,7 +61,10 @@ nav: - Flex Cell: user/examples/flex-cell/README.md - Incubator: user/examples/incubator/README.md - Firefighters: user/examples/o5g/README.md + - Mass Spring Damper with NuRV: user/examples/mass-spring-damper-monitor/README.md + - Incubator with NuRV: user/examples/incubator-NuRV-monitor-validation/README.md - Incubator with NuRV Service: user/examples/incubator-NuRV-monitor-service/README.md + - Water Tank Fault Injection with NuRV: user/examples/water_tank_FI_monitor/README.md - Codebase: https://github.com/INTO-CPS-Association/DTaaS-examples - FAQ: FAQ.md - Developer: diff --git a/servers/lib/HTTP.md b/servers/lib/HTTP.md new file mode 100644 index 000000000..30d30d8a7 --- /dev/null +++ b/servers/lib/HTTP.md @@ -0,0 +1,12 @@ +# HTTP File Server + +Endpoint: `localhost:PORT/lib/files` + +This option needs to be enabled with `-H http.json` flag. +If enabled, the 'localhost:PORT/lib/files' becomes available. +The regular file upload and download options become available. +Here are sample screenshots. + +![top-level](top-level.png) + +![inside](inside.png) diff --git a/servers/lib/README.md b/servers/lib/README.md index de42a1f9a..e3ab8528a 100644 --- a/servers/lib/README.md +++ b/servers/lib/README.md @@ -168,14 +168,8 @@ two end points: Endpoint: `localhost:PORT/lib/files` -This option needs to be enabled with `-H config.json` flag. -If enabled, the 'localhost:PORT/lib/files' becomes available. +This option needs to be enabled with `-H http.json` flag. The regular file upload and download options become available. -Here are sample screenshots. - -![top-level](top-level.png) - -![inside](inside.png) ### GraphQL protocol