diff --git a/.codeclimate.yml b/.codeclimate.yml index de8c1ce5f..144f6c2fe 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -20,7 +20,7 @@ plugins: fixme: enabled: true markdownlint: - enabled: true + enabled: true shellcheck: enabled: true diff --git a/.mdl_style.rb b/.mdl_style.rb new file mode 100644 index 000000000..ba34fb48e --- /dev/null +++ b/.mdl_style.rb @@ -0,0 +1,4 @@ +all +rule 'MD013', :ignore_code_blocks => true +rule 'MD013', :tables => false +rule 'MD046', :indented => true diff --git a/.mdlrc b/.mdlrc index 5fbe502bd..1f82ca2ce 100644 --- a/.mdlrc +++ b/.mdlrc @@ -1,5 +1 @@ -git_recurse true -all -rule 'MD013', :ignore_code_blocks => true -rule 'MD013', :tables => false -rule 'MD046', :indented \ No newline at end of file +style '.mdl_style.rb' diff --git a/README.md b/README.md index 94a8d9b35..2a75f20c7 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,29 @@ -# Motivation +# :factory: :left_right_arrow: :busts_in_silhouette: Digital Twin as a Service -The Digital Twin as a Service (DTaaS) software -is useful to create and run digital twins. -The digital twins that are running can be -used as service by other users. -These users need not be members of -the DTaaS software platform itself. +## :grinning: Motivation -There is an overview of the software available for: +The Digital Twin as a Service (DTaaS) software platform is useful +to **Build, Use and Share** digital twins (DTs). -* General users - - [overview slides](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/DTaaS-short-intro.pdf) - and - [overview video](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/DTaaS-short-intro.mp4), - [feature walkthrough](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/dtaas-v0.2.0-demo.mp4) -* Developers - [slides](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/DTaaS-overview.pdf) - and [video](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/DTaaS-overview.mkv). +**Build**: The DTs are built on the software platform +using the reusable DT components available on the platform. -There is also a -[research paper draft](https://arxiv.org/abs/2305.07244) -if you are interested in -reading the scientific roadmap for this software. +**Use**: Use the DTs on the software platform. + +**Share**: Share ready to use DTs with other users. +It is also possible to share the services offered by one DT with other users. + +## :rocket: Install and Use + +Please use the latest release available on +the [releases page](https://github.com/INTO-CPS-Association/DTaaS/releases) +and its [documentation](https://into-cps-association.github.io/DTaaS/) +to install and use the DTaaS software platform. + +You are welcome to open an [issue](https://github.com/INTO-CPS-Association/DTaaS/issues/new/choose) +if there is a suggestion to improve the software. + +## :hammer_and_wrench: Development Setup This is a mono repo containing code for both the web client and the microservices code base. @@ -29,83 +32,11 @@ Only the [web client](client) and components are functional at present. Everything else is a work-in-progress. -## Documentation - -The software comes with -[documentation](https://into-cps-association.github.io/DTaaS/) -for administrators and users. -You are welcome to open an issue -if there is a suggestion on improving the documentation. - -## Installation - -The software can be installed either on -Ubuntu Server 22.04 Operating System or -on vagrant virtual machine(s). -The installation instructions and scripts in `deploy/` -should help you get started. -If you face any issues, -please open an -[issue](https://github.com/INTO-CPS-Association/DTaaS/issues/new/choose). - -Some of the services like InfluxDB -require a dedicated hostname. -Thus successful installation of these services -is dependent on your ability to use -multiple hostnames for different services. -There are dedicated installation scripts -for services in `script/`. - -## Development Setup - -The rest of the information on this page -is aimed at current and potential contributors -to DTaaS software development. - -To install the development environment, run - -```bash -bash script/env.sh -``` - -There is a script to download all the docker containers used in the project. -You can download them using - -```bash -bash script/docker.sh -``` - -**CAVEAT**: The docker images are large and are likely to consume -about 5GB of bandwidth and 15GB of space. -You will have to download the docker images on a really good network. - -Before you make commits, please install the git hooks provided in the repository. - -```bash -bash script/configure-git-hooks.sh -``` - -This will ensure that your commits are formatted -correctly and that the unittests pass before you -push your changes. -Be aware that the tests take a long time to run. -If you want to skip the tests or formatting, -you can use the `--no-verify` flag -on `git commit` or `git push`. - -### Infrastructure Components - -The application uses -[Træfik](https://github.com/traefik/traefik) and -[ML Workspace](https://github.com/ml-tooling/ml-workspace) -open-source components. -It is possible to run jupyterlab notebooks, -[Grafana servers](script/grafana.sh), -[InfluxDB](script/influx.sh) and -[RabbitMQ](https://github.com/rabbitmq/rabbitmq-server) -as part of the DTaaS software. +Please see the +[developer documentation](https://into-cps-association.github.io/DTaaS/development/developer/index.html) +for more details. -## License +## :balance_scale: License This software is owned by [The INTO-CPS Association](https://into-cps.org/) diff --git a/client/.madgerc b/client/.madgerc new file mode 100644 index 000000000..4425cb8ef --- /dev/null +++ b/client/.madgerc @@ -0,0 +1,18 @@ +{ + "fontSize": "10px", + "tsconfig": "tsconfig.json", + "fileExtensions": ["ts", "tsx", "js", "jsx"], + "backgroundColor": "#FFFFFF", + "textColor": "#FFFFFF", + "nodeColor": "black", + "noDependencyColor": "green", + "cyclicNodeColor": "red", + "edgeColor": "green", + "graphVizOptions": { + "G": { + "rankdir": "TB", + "layout": "neato", + "splines": "curved" + } + } +} \ No newline at end of file diff --git a/client/README.md b/client/README.md index 5333073e5..9801dd526 100644 --- a/client/README.md +++ b/client/README.md @@ -13,6 +13,7 @@ cd client yarn install #install the nodejs dependencies yarn format #format .ts[x] and .js[x] files with prettier. yarn syntax #perform linting and static analysis +yarn graph # generate dependency graphs in the code yarn build #build the react app into build/ directory yarn develop #start the development server without building. Great for live edits. @@ -24,18 +25,23 @@ yarn start #start the application yarn clean #clean the directory of temporary files ``` -It is also possible to run different types of tests using the yarn test command by passing different flags: +It is also possible to run different types of tests using the yarn +test command by passing different flags: + ```bash yarn test -a #run all tests yarn test -u #run unit tests yarn test -i #run integration tests yarn test -e #run end-to-end tests ``` + --- ## Authentication -The react client website uses OAuth authentication. The [authentication page](../docs/admin/client/auth.md) provides details on setting up oauth authentication for the client application. +The react client website uses OAuth authentication. +The [authentication page](../docs/admin/client/auth.md) provides details +on setting up oauth authentication for the client application. ## Custom configuration diff --git a/client/package.json b/client/package.json index e007508f9..7e715d2d8 100644 --- a/client/package.json +++ b/client/package.json @@ -20,7 +20,8 @@ "configapp": "script/config.bash", "start": "script/start.bash", "clean": "script/clean.bash", - "format": "prettier --ignore-path ../.gitignore --write \"**/*.{ts,tsx,css,scss}\"" + "format": "prettier --ignore-path ../.gitignore --write \"**/*.{ts,tsx,css,scss}\"", + "graph": "script/graph.bash" }, "eslintConfig": { "extends": [ diff --git a/client/script/clean.bash b/client/script/clean.bash index 04e7279ef..222a856e8 100755 --- a/client/script/clean.bash +++ b/client/script/clean.bash @@ -1,2 +1,3 @@ #!/bin/bash -rm -rf build/ node_modules/ coverage/ playwright-report/ \ No newline at end of file +rm -rf build/ node_modules/ coverage/ playwright-report/ +rm src.svg test.svg \ No newline at end of file diff --git a/client/script/graph.bash b/client/script/graph.bash new file mode 100755 index 000000000..57f177c8e --- /dev/null +++ b/client/script/graph.bash @@ -0,0 +1,7 @@ +#!/bin/bash +PATH="$(yarn bin):$PATH" +export PATH +printf "Generate dependency graph for code" + +madge --image src.svg src +madge --image test.svg test diff --git a/deploy/README.md b/deploy/README.md index 9ab28fc28..143843c4b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -114,4 +114,4 @@ You can run this script multiple times until the installation is successful. ## Access the application -Now you should be able to access the DTaaS application at: _https://foo.com_ +Now you should be able to access the DTaaS application at: _https://foo.com_ diff --git a/docs/FAQ.md b/docs/FAQ.md index a5c689b7f..d57f69ec9 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,3 +1,4 @@ +# Frequently Asked Questions ## Abreviations @@ -40,7 +41,8 @@ DTaaS is not a model creation tool. You can put model creation tool inside DTaaS and create new models. - The DTaaS itself does not create digital twin models. But you can run + The DTaaS itself does not create digital twin models but it can help + users create digital twin models. You can run Linux desktop / terminal tools inside the DTaaS. So you can create models inside DTaaS and run them using tools that can run in Linux. The Windows only tools can not run in DTaaS. @@ -74,8 +76,9 @@ to provide one. ??? Question "Does it support XML-based representation and ontology representation?" - Currently No. **We are looking for users needing this capability. If you have concrete requirements and an example, we can discuss a way of realizing it in DTaaS**. - + Currently No. **We are looking for users needing this capability.** + **If you have concrete requirements and an example, we can discuss a way** + **of realizing it in DTaaS**. ## Communication Between Physical Twin and Digital Twin diff --git a/docs/admin/services.md b/docs/admin/services.md new file mode 100644 index 000000000..998a467ba --- /dev/null +++ b/docs/admin/services.md @@ -0,0 +1,128 @@ +# :electric_plug: Third-party Services + +The DTaaS software platform uses third-party software services +to provide enhanced value to users. + +InfluxDB, RabbitMQ and Grafana are default services +integrated into the DTaaS software platform. + +_The InfluxDB service requires a dedicated hostname. The management +interface of RabbitMQ service requires a dedicated hostname as well._ + +Thus successful installation of these services +is dependent on your ability to use +multiple hostnames for different services. You can download the required +services using the docker commands. + +```sh +docker pull traefik:v2.5 +docker pull influxdb:2.4 +docker pull mltooling/ml-workspace:0.13.2 +docker pull grafana/grafana +docker pull gitlab/gitlab-ce:15.10.0-ce.0 +``` + +:warning: The docker images are large and are likely to consume +about 5GB of bandwidth and 15GB of space. +You will have to download the docker images on a really good network. + +The two-machine vagrant deployment scenario installs the RabbitMQ, Grafana, and +InfluxDB services on the second vagrant machine. + +If you would like to install some of these services for native OS +installation or single vagrant machine, you can do this as well. + +## RabbitMQ Service + +Start the RabbitMQ service with + +```bash +docker run -d \ + --name rabbitmq-server \ + -p 15672:15672 -p 5672:5672 \ + rabbitmq:3-management +``` + +Users and the vhosts need to be setup on the server. Sample commands to do so are: + +```bash +docker exec rabbitmq-server rabbitmqctl add_user +docker exec rabbitmq-server rabbitmqctl set_permissions -p "/" ".*" ".*" ".*" +``` + +The RabbitMQ service requires raw TCP/UDP protocol access to network. +The default Traefik configuration of DTaaS does not permit TCP/UDP traffic. There are two possible choices here: + +* Configure Traefik gateway to permit TCP/UDP traffic +* Bypass Traefik altogether for RabbitMQ service + +Unless you are an informed user of Traefik, we recommend bypassing traefik +for RabbitMQ service. + +## Grafana Service + +Grafana service can run well behind Traefik gateway. Here is a sample docker +command to run Grafana service at port 3000: + +```bash +docker run -d \ + -p 3000:3000 \ + --name=grafana \ + -e "GF_SERVER_SERVE_FROM_SUB_PATH=true" \ + -e "GF_SERVER_DOMAIN=localhost" \ + -e "GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s:%(http_port)s" \ + -e "GF_AUTH_BASIC_ENABLED=false" \ + -e "GF_AUTH_PROXY_ENABLED=false" \ + -e "GF_SECURITY_ADMIN_PASSWORD=DTaaSGrafana" \ + -e "GF_SECURITY_ALLOW_EMBEDDING=true" \ + -e "GF_SECURITY_ALLOW_EMBEDDING=true" \ + -e "GF_AUTH_ANONYMOUS_ENABLED=true" \ + -e "GF_AUTH_ANONYMOUS_ORG_NAME=Main" \ + -e "GF_AUTH_ANONYMOUS_ORG_ROLE=Editor" \ + -e "GF_USERS_ALLOW_SIGN_UP=false" \ + -e "GF_FEATURE_TOGGLES_ENABLE=publicDashboards" \ + -e "GF_PATHS_CONFIG=/etc/grafana/grafana.ini" \ + -e "GF_PATHS_DATA=/var/lib/grafana" \ + -e "GF_PATHS_HOME=/usr/share/grafana" \ + -e "GF_PATHS_LOGS=/var/log/grafana" \ + -e "GF_PATHS_PLUGINS=/var/lib/grafana/plugins" \ + -e "GF_PATHS_PROVISIONING=/etc/grafana/provisioning" \ + -e "HOME=/home/grafana" \ + grafana/grafana +printf "Complete the setup from GUI" +``` + +The user credentials have also been set in the command as: + +**username**: admin +**password**: DTaaSGrafana + +Remember to change these credentials before starting the docker container. + +## InfluxDB Service + +The barebones InfluxDB service can be installed using: + +```bash +INFLUXDB_DATA="${PWD}/data/influxdb2" +mkdir -p "$INFLUXDB_DATA" + +# Remember to change the settings +docker run -d -p 80:8086 \ + --name influxdb24 \ + -v "$INFLUXDB_DATA/data":/var/lib/influxdb2 \ + -v "$INFLUXDB_DATA/config":/etc/influxdb2 \ + -e DOCKER_INFLUXDB_INIT_MODE=setup \ + -e DOCKER_INFLUXDB_INIT_USERNAME=dtaas \ + -e DOCKER_INFLUXDB_INIT_PASSWORD=dtaas1357 \ + -e DOCKER_INFLUXDB_INIT_ORG=dtaas \ + -e DOCKER_INFLUXDB_INIT_BUCKET=dtaas \ + influxdb:2.4 +``` + +The user credentials have also been set in the command as: + +**username**: dtaas +**password**: dtaas1357 + +Remember to change these credentials before starting the docker container. diff --git a/docs/admin/trial.md b/docs/admin/trial.md index 8e975ecea..6acab8eb7 100644 --- a/docs/admin/trial.md +++ b/docs/admin/trial.md @@ -1,6 +1,10 @@ # Trial Installation +The software can be installed either on +Ubuntu Server 22.04 Operating System or +on vagrant virtual machine(s). + A single step install script is helpful in performing a trial run of the software. This script installs DTaaS software with default credentials and users on a Ubuntu server OS. diff --git a/docs/developer/client/client.md b/docs/developer/client/client.md new file mode 100644 index 000000000..0f78d8fc4 --- /dev/null +++ b/docs/developer/client/client.md @@ -0,0 +1,19 @@ +# React Website + +The [Website](https://github.com/INTO-CPS-Association/DTaaS/tree/feature/distributed-demo/client#readme) +is how the end-users interact with the software platform. The website is +being developed as a React single page web application. + +A dependency graph for the entire codebase of the react application is: + +## Dependency Graphs + +The figures are the dependency graphs generated from the code. + +### src directory + +![src dependency graph](src.svg) + +### test directory + +![test dependency graph](test.svg) diff --git a/docs/developer/client/src.svg b/docs/developer/client/src.svg new file mode 100644 index 000000000..f7bbcfa16 --- /dev/null +++ b/docs/developer/client/src.svg @@ -0,0 +1,565 @@ + + + + + + +G + + + +AppProvider.tsx + +AppProvider.tsx + + + +route/auth/AuthProvider.tsx + +route/auth/AuthProvider.tsx + + + +AppProvider.tsx->route/auth/AuthProvider.tsx + + + + + +store/store.ts + +store/store.ts + + + +AppProvider.tsx->store/store.ts + + + + + +util/auth/useOidcConfig.ts + +util/auth/useOidcConfig.ts + + + +route/auth/AuthProvider.tsx->util/auth/useOidcConfig.ts + + + + + +store/menu.slice.ts + +store/menu.slice.ts + + + +store/store.ts->store/menu.slice.ts + + + + + +store/auth.slice.ts + +store/auth.slice.ts + + + +store/store.ts->store/auth.slice.ts + + + + + +components/Iframe.tsx + +components/Iframe.tsx + + + +components/LinkButtons.tsx + +components/LinkButtons.tsx + + + +components/LinkIconsLib.tsx + +components/LinkIconsLib.tsx + + + +components/LinkButtons.tsx->components/LinkIconsLib.tsx + + + + + +util/envUtil.ts + +util/envUtil.ts + + + +components/LinkButtons.tsx->util/envUtil.ts + + + + + +util/envUtil.ts->store/store.ts + + + + + +components/tab/TabComponent.tsx + +components/tab/TabComponent.tsx + + + +components/tab/subcomponents/TabRender.tsx + +components/tab/subcomponents/TabRender.tsx + + + +components/tab/TabComponent.tsx->components/tab/subcomponents/TabRender.tsx + + + + + +components/tab/subcomponents/TabStyles.ts + +components/tab/subcomponents/TabStyles.ts + + + +components/tab/TabComponent.tsx->components/tab/subcomponents/TabStyles.ts + + + + + +index.tsx + +index.tsx + + + +index.tsx->AppProvider.tsx + + + + + +index.tsx->util/envUtil.ts + + + + + +page/LayoutPublic.tsx + +page/LayoutPublic.tsx + + + +index.tsx->page/LayoutPublic.tsx + + + + + +route/auth/Account.tsx + +route/auth/Account.tsx + + + +index.tsx->route/auth/Account.tsx + + + + + +route/auth/PrivateRoute.tsx + +route/auth/PrivateRoute.tsx + + + +index.tsx->route/auth/PrivateRoute.tsx + + + + + +route/auth/Signin.tsx + +route/auth/Signin.tsx + + + +index.tsx->route/auth/Signin.tsx + + + + + +route/digitaltwins/DigitalTwins.tsx + +route/digitaltwins/DigitalTwins.tsx + + + +index.tsx->route/digitaltwins/DigitalTwins.tsx + + + + + +route/library/Library.tsx + +route/library/Library.tsx + + + +index.tsx->route/library/Library.tsx + + + + + +route/workbench/Workbench.tsx + +route/workbench/Workbench.tsx + + + +index.tsx->route/workbench/Workbench.tsx + + + + + +page/Footer.tsx + +page/Footer.tsx + + + +page/LayoutPublic.tsx->page/Footer.tsx + + + + + +page/Layout.tsx + +page/Layout.tsx + + + +route/auth/Account.tsx->page/Layout.tsx + + + + + +route/auth/AccountTabs.tsx + +route/auth/AccountTabs.tsx + + + +route/auth/Account.tsx->route/auth/AccountTabs.tsx + + + + + +route/auth/WaitAndNavigate.tsx + +route/auth/WaitAndNavigate.tsx + + + +route/auth/PrivateRoute.tsx->route/auth/WaitAndNavigate.tsx + + + + + +route/digitaltwins/DigitalTwins.tsx->components/Iframe.tsx + + + + + +route/digitaltwins/DigitalTwins.tsx->util/envUtil.ts + + + + + +route/digitaltwins/DigitalTwins.tsx->components/tab/TabComponent.tsx + + + + + +route/digitaltwins/DigitalTwins.tsx->components/tab/subcomponents/TabRender.tsx + + + + + +route/digitaltwins/DigitalTwins.tsx->page/Layout.tsx + + + + + +route/digitaltwins/DigitalTwinTabData.ts + +route/digitaltwins/DigitalTwinTabData.ts + + + +route/digitaltwins/DigitalTwins.tsx->route/digitaltwins/DigitalTwinTabData.ts + + + + + +route/library/Library.tsx->components/Iframe.tsx + + + + + +route/library/Library.tsx->util/envUtil.ts + + + + + +route/library/Library.tsx->components/tab/TabComponent.tsx + + + + + +route/library/Library.tsx->components/tab/subcomponents/TabRender.tsx + + + + + +route/library/Library.tsx->page/Layout.tsx + + + + + +util/auth/Authentication.ts + +util/auth/Authentication.ts + + + +route/library/Library.tsx->util/auth/Authentication.ts + + + + + +route/library/LibraryTabData.ts + +route/library/LibraryTabData.ts + + + +route/library/Library.tsx->route/library/LibraryTabData.ts + + + + + +route/workbench/Workbench.tsx->components/LinkButtons.tsx + + + + + +route/workbench/Workbench.tsx->util/envUtil.ts + + + + + +route/workbench/Workbench.tsx->page/Layout.tsx + + + + + +page/DrawerComponent.tsx + +page/DrawerComponent.tsx + + + +page/DrawerHeaderComponent.tsx + +page/DrawerHeaderComponent.tsx + + + +page/DrawerComponent.tsx->page/DrawerHeaderComponent.tsx + + + + + +page/MenuItems.tsx + +page/MenuItems.tsx + + + +page/DrawerComponent.tsx->page/MenuItems.tsx + + + + + +page/MenuToolbar.tsx + +page/MenuToolbar.tsx + + + +page/DrawerComponent.tsx->page/MenuToolbar.tsx + + + + + +page/MenuToolbar.tsx->util/auth/Authentication.ts + + + + + +page/Layout.tsx->page/Footer.tsx + + + + + +page/Menu.tsx + +page/Menu.tsx + + + +page/Layout.tsx->page/Menu.tsx + + + + + +page/Menu.tsx->store/store.ts + + + + + +page/Menu.tsx->page/DrawerComponent.tsx + + + + + +page/Menu.tsx->page/MenuToolbar.tsx + + + + + +page/Menu.tsx->store/menu.slice.ts + + + + + +util/auth/Authentication.ts->util/envUtil.ts + + + + + +util/auth/Authentication.ts->store/auth.slice.ts + + + + + +page/Title.tsx + +page/Title.tsx + + + +route/IData.ts + +route/IData.ts + + + +route/auth/AccountTabs.tsx->components/tab/TabComponent.tsx + + + + + +route/auth/AccountTabs.tsx->components/tab/subcomponents/TabRender.tsx + + + + + +util/auth/useOidcConfig.ts->util/envUtil.ts + + + + + +route/auth/WaitAndNavigate.tsx->util/auth/Authentication.ts + + + + + +route/digitaltwins/DigitalTwinTabData.ts->route/IData.ts + + + + + +route/library/LibraryTabData.ts->route/IData.ts + + + + + diff --git a/docs/developer/client/test.svg b/docs/developer/client/test.svg new file mode 100644 index 000000000..e651b2c84 --- /dev/null +++ b/docs/developer/client/test.svg @@ -0,0 +1,607 @@ + + + + + + +G + + + +../src/components/Iframe.tsx + +../src/components/Iframe.tsx + + + +../src/components/tab/TabComponent.tsx + +../src/components/tab/TabComponent.tsx + + + +../src/components/tab/subcomponents/TabRender.tsx + +../src/components/tab/subcomponents/TabRender.tsx + + + +../src/components/tab/TabComponent.tsx->../src/components/tab/subcomponents/TabRender.tsx + + + + + +../src/components/tab/subcomponents/TabStyles.ts + +../src/components/tab/subcomponents/TabStyles.ts + + + +../src/components/tab/TabComponent.tsx->../src/components/tab/subcomponents/TabStyles.ts + + + + + +../src/page/DrawerComponent.tsx + +../src/page/DrawerComponent.tsx + + + +../src/page/DrawerHeaderComponent.tsx + +../src/page/DrawerHeaderComponent.tsx + + + +../src/page/DrawerComponent.tsx->../src/page/DrawerHeaderComponent.tsx + + + + + +../src/page/MenuItems.tsx + +../src/page/MenuItems.tsx + + + +../src/page/DrawerComponent.tsx->../src/page/MenuItems.tsx + + + + + +../src/page/MenuToolbar.tsx + +../src/page/MenuToolbar.tsx + + + +../src/page/DrawerComponent.tsx->../src/page/MenuToolbar.tsx + + + + + +../src/util/auth/Authentication.ts + +../src/util/auth/Authentication.ts + + + +../src/page/MenuToolbar.tsx->../src/util/auth/Authentication.ts + + + + + +../src/page/Footer.tsx + +../src/page/Footer.tsx + + + +../src/page/Layout.tsx + +../src/page/Layout.tsx + + + +../src/page/Layout.tsx->../src/page/Footer.tsx + + + + + +../src/page/Menu.tsx + +../src/page/Menu.tsx + + + +../src/page/Layout.tsx->../src/page/Menu.tsx + + + + + +../src/page/Menu.tsx->../src/page/DrawerComponent.tsx + + + + + +../src/page/Menu.tsx->../src/page/MenuToolbar.tsx + + + + + +../src/store/menu.slice.ts + +../src/store/menu.slice.ts + + + +../src/page/Menu.tsx->../src/store/menu.slice.ts + + + + + +../src/store/store.ts + +../src/store/store.ts + + + +../src/page/Menu.tsx->../src/store/store.ts + + + + + +../src/store/store.ts->../src/store/menu.slice.ts + + + + + +../src/store/auth.slice.ts + +../src/store/auth.slice.ts + + + +../src/store/store.ts->../src/store/auth.slice.ts + + + + + +../src/util/envUtil.ts + +../src/util/envUtil.ts + + + +../src/util/auth/Authentication.ts->../src/util/envUtil.ts + + + + + +../src/util/auth/Authentication.ts->../src/store/auth.slice.ts + + + + + +../src/route/IData.ts + +../src/route/IData.ts + + + +../src/route/auth/AuthProvider.tsx + +../src/route/auth/AuthProvider.tsx + + + +../src/util/auth/useOidcConfig.ts + +../src/util/auth/useOidcConfig.ts + + + +../src/route/auth/AuthProvider.tsx->../src/util/auth/useOidcConfig.ts + + + + + +../src/util/auth/useOidcConfig.ts->../src/util/envUtil.ts + + + + + +../src/route/auth/PrivateRoute.tsx + +../src/route/auth/PrivateRoute.tsx + + + +../src/route/auth/WaitAndNavigate.tsx + +../src/route/auth/WaitAndNavigate.tsx + + + +../src/route/auth/PrivateRoute.tsx->../src/route/auth/WaitAndNavigate.tsx + + + + + +../src/route/auth/WaitAndNavigate.tsx->../src/util/auth/Authentication.ts + + + + + +../src/route/library/Library.tsx + +../src/route/library/Library.tsx + + + +../src/route/library/Library.tsx->../src/components/Iframe.tsx + + + + + +../src/route/library/Library.tsx->../src/components/tab/TabComponent.tsx + + + + + +../src/route/library/Library.tsx->../src/components/tab/subcomponents/TabRender.tsx + + + + + +../src/route/library/Library.tsx->../src/page/Layout.tsx + + + + + +../src/route/library/Library.tsx->../src/util/auth/Authentication.ts + + + + + +../src/route/library/LibraryTabData.ts + +../src/route/library/LibraryTabData.ts + + + +../src/route/library/Library.tsx->../src/route/library/LibraryTabData.ts + + + + + +../src/route/library/Library.tsx->../src/util/envUtil.ts + + + + + +../src/route/library/LibraryTabData.ts->../src/route/IData.ts + + + + + +../src/util/envUtil.ts->../src/store/store.ts + + + + + +e2e/playwright/Auth.test.ts + +e2e/playwright/Auth.test.ts + + + +e2e/playwright/Links.ts + +e2e/playwright/Links.ts + + + +e2e/playwright/Auth.test.ts->e2e/playwright/Links.ts + + + + + +e2e/playwright/Menu.test.ts + +e2e/playwright/Menu.test.ts + + + +e2e/playwright/Menu.test.ts->e2e/playwright/Links.ts + + + + + +e2e/playwright/auth.setup.ts + +e2e/playwright/auth.setup.ts + + + +integration/authRedux.test.tsx + +integration/authRedux.test.tsx + + + +integration/authRedux.test.tsx->../src/route/auth/PrivateRoute.tsx + + + + + +integration/authRedux.test.tsx->../src/route/library/Library.tsx + + + + + +integration/authRedux.test.tsx->../src/store/auth.slice.ts + + + + + +unitTests/testUtils.tsx + +unitTests/testUtils.tsx + + + +integration/authRedux.test.tsx->unitTests/testUtils.tsx + + + + + +testUtil.ts + +testUtil.ts + + + +unitTests/Components/Iframe.test.tsx + +unitTests/Components/Iframe.test.tsx + + + +unitTests/Components/Linkbuttons.test.tsx + +unitTests/Components/Linkbuttons.test.tsx + + + +unitTests/Components/PrivateRoute.test.tsx + +unitTests/Components/PrivateRoute.test.tsx + + + +unitTests/Components/PrivateRoute.test.tsx->../src/route/auth/PrivateRoute.tsx + + + + + +unitTests/Components/PrivateRoute.test.tsx->unitTests/testUtils.tsx + + + + + +unitTests/Components/TabComponent.test.tsx + +unitTests/Components/TabComponent.test.tsx + + + +unitTests/Page/Layout.test.tsx + +unitTests/Page/Layout.test.tsx + + + +unitTests/Page/page.testUtils.tsx + +unitTests/Page/page.testUtils.tsx + + + +unitTests/Page/Layout.test.tsx->unitTests/Page/page.testUtils.tsx + + + + + +unitTests/Page/page.testUtils.tsx->unitTests/testUtils.tsx + + + + + +unitTests/Page/LayoutPublic.test.tsx + +unitTests/Page/LayoutPublic.test.tsx + + + +unitTests/Page/LayoutPublic.test.tsx->unitTests/Page/page.testUtils.tsx + + + + + +unitTests/Routes/DigitalTwins.test.tsx + +unitTests/Routes/DigitalTwins.test.tsx + + + +unitTests/Routes/DigitalTwins.test.tsx->unitTests/testUtils.tsx + + + + + +unitTests/__mocks__/global_mocks.ts + +unitTests/__mocks__/global_mocks.ts + + + +unitTests/Routes/DigitalTwins.test.tsx->unitTests/__mocks__/global_mocks.ts + + + + + +unitTests/Routes/Library.test.tsx + +unitTests/Routes/Library.test.tsx + + + +unitTests/Routes/Library.test.tsx->../src/route/auth/AuthProvider.tsx + + + + + +unitTests/Routes/Library.test.tsx->unitTests/testUtils.tsx + + + + + +unitTests/Routes/Library.test.tsx->unitTests/__mocks__/global_mocks.ts + + + + + +unitTests/Routes/SignIn.test.tsx + +unitTests/Routes/SignIn.test.tsx + + + +unitTests/Routes/Workbench.test.tsx + +unitTests/Routes/Workbench.test.tsx + + + +unitTests/Routes/Workbench.test.tsx->unitTests/testUtils.tsx + + + + + +unitTests/Util/Store.test.ts + +unitTests/Util/Store.test.ts + + + +unitTests/Util/envUtil.test.ts + +unitTests/Util/envUtil.test.ts + + + +unitTests/__mocks__/component_mocks.tsx + +unitTests/__mocks__/component_mocks.tsx + + + +unitTests/__mocks__/page_mocks.tsx + +unitTests/__mocks__/page_mocks.tsx + + + +unitTests/auth/AuthProvider.test.tsx + +unitTests/auth/AuthProvider.test.tsx + + + +unitTests/auth/AuthProvider.test.tsx->../src/route/auth/AuthProvider.tsx + + + + + +unitTests/auth/AuthProvider.test.tsx->../src/util/auth/useOidcConfig.ts + + + + + +unitTests/jest.setup.ts + +unitTests/jest.setup.ts + + + +unitTests/jest.setup.ts->unitTests/__mocks__/global_mocks.ts + + + + + +unitTests/jest.setup.ts->unitTests/__mocks__/component_mocks.tsx + + + + + +unitTests/jest.setup.ts->unitTests/__mocks__/page_mocks.tsx + + + + + diff --git a/docs/developer/client/uml/package-diagram.puml b/docs/developer/client/uml/package-diagram.puml deleted file mode 100644 index 416d6cad3..000000000 --- a/docs/developer/client/uml/package-diagram.puml +++ /dev/null @@ -1,78 +0,0 @@ -@startuml -hide empty members - - -package "Routes" { - class "Workbench" - class "Library" - class "Digital Twins" as DT - - Workbench -l[hidden]- Library - Library -l[hidden]- DT -} - -package "Page" { - package "StructualComponent" { - class Layout - class Tabs - class PrivateRoute - - Layout -l[hidden]- Tabs - Tabs -l[hidden]- PrivateRoute - } - package "BaseComponent" { - class Footer - class Menu - class Title - - Title -l[hidden]- Menu - Menu -l[hidden]- Footer - } - - StructualComponent --[hidden] BaseComponent -} - -package "Util" { - class APIutil - class EnvUtil - - APIutil -l[hidden]- EnvUtil -} - -package "Components" { - class Iframe - class LinkButtons - class ShoppingCart - class AssetBoard - - Iframe -l[hidden] LinkButtons - LinkButtons -l[hidden] ShoppingCart - ShoppingCart -l[hidden] AssetBoard -} - -package "Store" { - class AuthAcess { - + useAuthToken() - } - class AppAcess { - + useApp() - } - class CartAcess { - + useCart() - } - class UserAcess { - + useUserData() - } - - AuthAcess -l[hidden] AppAcess - AppAcess -l[hidden] CartAcess - CartAcess -l[hidden] UserAcess -} - -Routes ..> Page : <> -Routes ..> Util : <> -Routes ..> Components : <> -Util ..> Store -Components ..> Util : <> -Components ..> Store : <> -@enduml diff --git a/docs/developer/index.md b/docs/developer/index.md new file mode 100644 index 000000000..217107f26 --- /dev/null +++ b/docs/developer/index.md @@ -0,0 +1,100 @@ +# :technologist: Developers Guide + +This guide is to help developers get familiar with the project. Please see +developer-specific +[Slides](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/DTaaS-overview.pdf), +[Video](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/DTaaS-overview.mp4), +and [Research paper](https://arxiv.org/abs/2305.07244). + +## :computer: Development Environment + +Ideally, developers should work on Ubuntu/Linux. Other operating systems +are not supported inherently and may require additional steps. + +To start with, install the required software and git-hooks. + +```bash +bash script/env.sh +bash script/configure-git-hooks.sh +``` + +The git-hooks will ensure that your commits are formatted +correctly and that the tests pass before you +push the commits to remote repositories. + +Be aware that the tests may take a long time to run. +If you want to skip the tests or formatting, +you can use the `--no-verify` flag +on `git commit` or `git push`. Please use this +option with care. + +There is a script to download all the docker containers +used in the project. You can download them using + +```bash +bash script/docker.sh +``` + +:warning: The docker images are large and are likely to consume +about 5GB of bandwidth and 15GB of space. +You will have to download the docker images on a really good network. + +## :building_construction: Development Workflow + +To manage collaboration by multiple developers on the software, +a development workflow is in place. Each developer should follow these steps: + +1. Fork of the main repository into your github account. +1. Setup +[Code Climate](https://docs.codeclimate.com/docs/getting-started-with-code-climate) +and +[Codecov](https://docs.codecov.com/docs/quick-start) +for your fork. The codecov does not require secret token +for public repositories. +1. Install git-hooks for the project. +1. Use +[Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/) +workflow. +1. Work in your fork and open a PR from your working branch to your `feature/distributed-demo` branch. +The PR will run all the github actions, code climate and codecov checks. +1. Resolve all the issues identified in the previous step. +1. If you have access to the +[integration server](https://github.com/INTO-CPS-Association/DTaaS/wiki/DTaaS-Integration-Server), +try your working branch on the integration server. +1. Once changes are verified, a PR should be made to the `feature/distributed-demo` branch of +the upstream +[DTaaS repository](https://github.com/into-cps-association/DTaaS). +1. The PR will be merged after checks by either the project administrators or the maintainers. + +Remember that every PR should be meaningful and satisfies a well-defined user story or improve +the code quality. + +## :eye: Code Quality + +The project code qualities are measured based on: + +* Linting issues identified by +[Code Climate](https://codeclimate.com/github/INTO-CPS-Association/DTaaS) +* Test coverage report collected by +[Codecov](https://codecov.io/gh/INTO-CPS-Association/DTaaS) +* Successful [github actions](https://github.com/INTO-CPS-Association/DTaaS/actions) + +### Code Climate + +Code Climate performs static analysis, linting and style checks. Quality checks are performed by codeclimate are to ensure the best possible quality of code to add to our project. + +While any new issues introduced in your code would be shown in the PR page itself, to address any specific issue, you can visit the issues or code section of the codeclimate page. + +It is highly recommended that any code you add does not introduce new quality issues. If they are introduced, they should be fixed immediately using the appropriate suggestions from Code Climate, or in worst case, adding a ignore flag (To be used with caution). + +### Codecov + +Codecov keeps track of the test coverage for the entire project. +For information about testing and workflow related to that, please see the [testing page](testing/intro.md). + +### Github Actions + +The project has multiple +[github actions](https://github.com/INTO-CPS-Association/DTaaS/tree/feature/distributed-demo/.github/workflows) +defined. All PRs and direct code commits must have successful +status on github actions. diff --git a/docs/developer/servers/lib/lib-class.mmd b/docs/developer/servers/lib/lib-class.mmd new file mode 100644 index 000000000..93eed3779 --- /dev/null +++ b/docs/developer/servers/lib/lib-class.mmd @@ -0,0 +1,46 @@ +classDiagram + class FilesResolver { + -filesService: IFilesService + +listDirectory(path: string): Promise + +readFile(path: string): Promise + } + + class FilesServiceFactory { + -configService: ConfigService + -gitlabFilesService: GitlabFilesService + -localFilesService: LocalFilesService + +create(): IFilesService + } + + class GitlabFilesService { + -configService: ConfigService + -parseArguments(path: string): Promise + -sendRequest(query: string): Promise + -executeQuery(path: string, getQuery: QueryFunction): Promise + +listDirectory(path: string): Promise + +readFile(path: string): Promise + } + + class LocalFilesService { + -configService: ConfigService + -getFileStats(fullPath: string, file: string): Promise + +listDirectory(path: string): Promise + +readFile(path: string): Promise + } + + class ConfigService { + +get(propertyPath: string): any + } + + class IFilesService{ + listDirectory(path: string): Promise + readFile(path: string): Promise + } + + IFilesService <|-- FilesResolver: uses + IFilesService <|.. GitlabFilesService: implements + IFilesService <|.. LocalFilesService: implements + IFilesService <|-- FilesServiceFactory: creates + ConfigService <|-- FilesServiceFactory: uses + ConfigService <|-- GitlabFilesService: uses + ConfigService <|-- LocalFilesService: uses \ No newline at end of file diff --git a/docs/developer/servers/lib/lib-ms.md b/docs/developer/servers/lib/lib-ms.md new file mode 100644 index 000000000..740b93e55 --- /dev/null +++ b/docs/developer/servers/lib/lib-ms.md @@ -0,0 +1,189 @@ +# Library Microservice + +[The Library Microservices](https://github.com/INTO-CPS-Association/DTaaS/tree/feature/distributed-demo/servers/lib#readme) - +provides users with access to files in user workspaces via API. +This microservice will interface with local file system and Gitlab +to provide uniform Gitlab-compliant API access to files. + +!!! warning + This microservice is still under heavy development. It is still not + a good replacement for file server we are using now. + +## Architecture and Design + +The C4 level 2 diagram of this microservice is: + +Library Microservice + +The GraphQL API provided by the library microservice shall be compliant +with the Gitlab GraphQL service. + +## UML Diagrams + +### Class Diagram + +```mermaid +classDiagram + class FilesResolver { + -filesService: IFilesService + +listDirectory(path: string): Promise + +readFile(path: string): Promise + } + + class FilesServiceFactory { + -configService: ConfigService + -gitlabFilesService: GitlabFilesService + -localFilesService: LocalFilesService + +create(): IFilesService + } + + class GitlabFilesService { + -configService: ConfigService + -parseArguments(path: string): Promise + -sendRequest(query: string): Promise + -executeQuery(path: string, getQuery: QueryFunction): Promise + +listDirectory(path: string): Promise + +readFile(path: string): Promise + } + + class LocalFilesService { + -configService: ConfigService + -getFileStats(fullPath: string, file: string): Promise + +listDirectory(path: string): Promise + +readFile(path: string): Promise + } + + class ConfigService { + +get(propertyPath: string): any + } + + class IFilesService{ + listDirectory(path: string): Promise + readFile(path: string): Promise + } + + IFilesService <|-- FilesResolver: uses + IFilesService <|.. GitlabFilesService: implements + IFilesService <|.. LocalFilesService: implements + IFilesService <|-- FilesServiceFactory: creates + ConfigService <|-- FilesServiceFactory: uses + ConfigService <|-- GitlabFilesService: uses + ConfigService <|-- LocalFilesService: uses +``` + +### Sequence Diagram + +```mermaid +sequenceDiagram + actor Client + actor Traefik + + box LightGreen Library Microservice + participant FR as FilesResolver + participant FSF as FilesServiceFactory + participant CS as ConfigService + participant IFS as IFilesService + participant LFS as LocalFilesService + participant GFS as GitlabFilesService + end + + participant FS as Local File System DB + participant GAPI as GitLab API DB + + Client ->> Traefik : HTTP request + Traefik ->> FR : GraphQL query + activate FR + + FR ->> FSF : create() + activate FSF + + FSF ->> CS : getConfiguration("MODE") + activate CS + + CS -->> FSF : return configuration value + deactivate CS + + alt MODE = Local + FSF ->> FR : return filesService (LFS) + deactivate FSF + + FR ->> IFS : listDirectory(path) or readFile(path) + activate IFS + + IFS ->> LFS : listDirectory(path) or readFile(path) + activate LFS + + LFS ->> CS : getConfiguration("LOCAL_PATH") + activate CS + + CS -->> LFS : return local path + deactivate CS + + LFS ->> FS : Access filesystem + alt Filesystem error + FS -->> LFS : Filesystem error + LFS ->> LFS : Throw new InternalServerErrorException + LFS -->> IFS : Error + else Successful file operation + FS -->> LFS : Return filesystem data + LFS ->> IFS : return Promise + end + deactivate LFS + else MODE = GitLab + FSF ->> FR : return filesService (GFS) + %%deactivate FSF + + FR ->> IFS : listDirectory(path) or readFile(path) + activate IFS + + IFS ->> GFS : listDirectory(path) or readFile(path) + activate GFS + + GFS ->> GFS : parseArguments(path) + GFS ->> GFS : executeQuery() + + GFS ->> CS : getConfiguration("GITLAB_API_URL", "GITLAB_TOKEN") + activate CS + + CS -->> GFS : return GitLab API URL and Token + deactivate CS + + GFS ->> GAPI : sendRequest() + alt GitLab API error + GAPI -->> GFS : API error + GFS ->> GFS : Throw new Error("Invalid query") + GFS -->> IFS : Error + else Successful GitLab API operation + GAPI -->> GFS : Return API response + GFS ->> IFS : return Promise + end + deactivate GFS + end + + alt Error thrown + IFS ->> FR : return Error + deactivate IFS + FR ->> Traefik : return Error + Traefik ->> Client : HTTP error response + else Successful operation + IFS ->> FR : return Promise + deactivate IFS + FR ->> Traefik : return Promise + Traefik ->> Client : HTTP response + end + + deactivate FR + +``` + +## Dependency Graphs + +The figures are the dependency graphs generated from the code. + +### src directory + +![src dependency graph](src.svg) + +### test directory + +![test dependency graph](test.svg) diff --git a/docs/developer/servers/lib/lib-ms.png b/docs/developer/servers/lib/lib-ms.png new file mode 100644 index 000000000..c6303a7a4 Binary files /dev/null and b/docs/developer/servers/lib/lib-ms.png differ diff --git a/docs/developer/servers/lib/lib-sequence.mmd b/docs/developer/servers/lib/lib-sequence.mmd new file mode 100644 index 000000000..68d95b654 --- /dev/null +++ b/docs/developer/servers/lib/lib-sequence.mmd @@ -0,0 +1,99 @@ +sequenceDiagram + actor Client + actor Traefik + + box LightGreen RAMS + participant FR as FilesResolver + participant FSF as FilesServiceFactory + participant CS as ConfigService + participant IFS as IFilesService + participant LFS as LocalFilesService + participant GFS as GitlabFilesService + end + + participant FS as Local File System DB + participant GAPI as GitLab API DB + + Client ->> Traefik : HTTP request + Traefik ->> FR : GraphQL query + activate FR + + FR ->> FSF : create() + activate FSF + + FSF ->> CS : getConfiguration("MODE") + activate CS + + CS -->> FSF : return configuration value + deactivate CS + + alt MODE = Local + FSF ->> FR : return filesService (LFS) + deactivate FSF + + FR ->> IFS : listDirectory(path) or readFile(path) + activate IFS + + IFS ->> LFS : listDirectory(path) or readFile(path) + activate LFS + + LFS ->> CS : getConfiguration("LOCAL_PATH") + activate CS + + CS -->> LFS : return local path + deactivate CS + + LFS ->> FS : Access filesystem + alt Filesystem error + FS -->> LFS : Filesystem error + LFS ->> LFS : Throw new InternalServerErrorException + LFS -->> IFS : Error + else Successful file operation + FS -->> LFS : Return filesystem data + LFS ->> IFS : return Promise + end + deactivate LFS + else MODE = GitLab + FSF ->> FR : return filesService (GFS) + %%deactivate FSF + + FR ->> IFS : listDirectory(path) or readFile(path) + activate IFS + + IFS ->> GFS : listDirectory(path) or readFile(path) + activate GFS + + GFS ->> GFS : parseArguments(path) + GFS ->> GFS : executeQuery() + + GFS ->> CS : getConfiguration("GITLAB_API_URL", "GITLAB_TOKEN") + activate CS + + CS -->> GFS : return GitLab API URL and Token + deactivate CS + + GFS ->> GAPI : sendRequest() + alt GitLab API error + GAPI -->> GFS : API error + GFS ->> GFS : Throw new Error("Invalid query") + GFS -->> IFS : Error + else Successful GitLab API operation + GAPI -->> GFS : Return API response + GFS ->> IFS : return Promise + end + deactivate GFS + end + + alt Error thrown + IFS ->> FR : return Error + deactivate IFS + FR ->> Traefik : return Error + Traefik ->> Client : HTTP error response + else Successful operation + IFS ->> FR : return Promise + deactivate IFS + FR ->> Traefik : return Promise + Traefik ->> Client : HTTP response + end + + deactivate FR diff --git a/docs/developer/servers/lib/src.svg b/docs/developer/servers/lib/src.svg new file mode 100644 index 000000000..21aabe4d7 --- /dev/null +++ b/docs/developer/servers/lib/src.svg @@ -0,0 +1,193 @@ + + + + + + +G + + + +../util.ts + +../util.ts + + + +app.module.ts + +app.module.ts + + + +app.module.ts->../util.ts + + + + + +files/files.module.ts + +files/files.module.ts + + + +app.module.ts->files/files.module.ts + + + + + +files/resolvers/files.resolver.ts + +files/resolvers/files.resolver.ts + + + +files/files.module.ts->files/resolvers/files.resolver.ts + + + + + +files/services/files-service.factory.ts + +files/services/files-service.factory.ts + + + +files/files.module.ts->files/services/files-service.factory.ts + + + + + +files/services/gitlab-files.service.ts + +files/services/gitlab-files.service.ts + + + +files/files.module.ts->files/services/gitlab-files.service.ts + + + + + +files/services/local-files.service.ts + +files/services/local-files.service.ts + + + +files/files.module.ts->files/services/local-files.service.ts + + + + + +files/resolvers/files.resolver.ts->files/services/files-service.factory.ts + + + + + +files/interfaces/files.service.interface.ts + +files/interfaces/files.service.interface.ts + + + +files/resolvers/files.resolver.ts->files/interfaces/files.service.interface.ts + + + + + +types.ts + +types.ts + + + +files/resolvers/files.resolver.ts->types.ts + + + + + +files/services/files-service.factory.ts->files/services/gitlab-files.service.ts + + + + + +files/services/files-service.factory.ts->files/services/local-files.service.ts + + + + + +files/services/files-service.factory.ts->files/interfaces/files.service.interface.ts + + + + + +files/services/gitlab-files.service.ts->files/interfaces/files.service.interface.ts + + + + + +files/services/gitlab-files.service.ts->types.ts + + + + + +files/queries.ts + +files/queries.ts + + + +files/services/gitlab-files.service.ts->files/queries.ts + + + + + +files/services/local-files.service.ts->files/interfaces/files.service.interface.ts + + + + + +files/services/local-files.service.ts->types.ts + + + + + +files/interfaces/files.service.interface.ts->types.ts + + + + + +main.ts + +main.ts + + + +main.ts->app.module.ts + + + + + diff --git a/docs/developer/servers/lib/test.svg b/docs/developer/servers/lib/test.svg new file mode 100644 index 000000000..7d86c4853 --- /dev/null +++ b/docs/developer/servers/lib/test.svg @@ -0,0 +1,337 @@ + + + + + + +G + + + +../src/app.module.ts + +../src/app.module.ts + + + +../src/files/files.module.ts + +../src/files/files.module.ts + + + +../src/app.module.ts->../src/files/files.module.ts + + + + + +../util.ts + +../util.ts + + + +../src/app.module.ts->../util.ts + + + + + +../src/files/resolvers/files.resolver.ts + +../src/files/resolvers/files.resolver.ts + + + +../src/files/files.module.ts->../src/files/resolvers/files.resolver.ts + + + + + +../src/files/services/files-service.factory.ts + +../src/files/services/files-service.factory.ts + + + +../src/files/files.module.ts->../src/files/services/files-service.factory.ts + + + + + +../src/files/services/gitlab-files.service.ts + +../src/files/services/gitlab-files.service.ts + + + +../src/files/files.module.ts->../src/files/services/gitlab-files.service.ts + + + + + +../src/files/services/local-files.service.ts + +../src/files/services/local-files.service.ts + + + +../src/files/files.module.ts->../src/files/services/local-files.service.ts + + + + + +../src/files/resolvers/files.resolver.ts->../src/files/services/files-service.factory.ts + + + + + +../src/files/interfaces/files.service.interface.ts + +../src/files/interfaces/files.service.interface.ts + + + +../src/files/resolvers/files.resolver.ts->../src/files/interfaces/files.service.interface.ts + + + + + +../src/types.ts + +../src/types.ts + + + +../src/files/resolvers/files.resolver.ts->../src/types.ts + + + + + +../src/files/services/files-service.factory.ts->../src/files/services/gitlab-files.service.ts + + + + + +../src/files/services/files-service.factory.ts->../src/files/services/local-files.service.ts + + + + + +../src/files/services/files-service.factory.ts->../src/files/interfaces/files.service.interface.ts + + + + + +../src/files/services/gitlab-files.service.ts->../src/files/interfaces/files.service.interface.ts + + + + + +../src/files/services/gitlab-files.service.ts->../src/types.ts + + + + + +../src/files/queries.ts + +../src/files/queries.ts + + + +../src/files/services/gitlab-files.service.ts->../src/files/queries.ts + + + + + +../src/files/services/local-files.service.ts->../src/files/interfaces/files.service.interface.ts + + + + + +../src/files/services/local-files.service.ts->../src/types.ts + + + + + +../src/files/interfaces/files.service.interface.ts->../src/types.ts + + + + + +e2e/app.e2e.spec.ts + +e2e/app.e2e.spec.ts + + + +e2e/app.e2e.spec.ts->../src/app.module.ts + + + + + +testUtil.ts + +testUtil.ts + + + +e2e/app.e2e.spec.ts->testUtil.ts + + + + + +integration/files.service.integration.spec.ts + +integration/files.service.integration.spec.ts + + + +integration/files.service.integration.spec.ts->../src/files/resolvers/files.resolver.ts + + + + + +integration/files.service.integration.spec.ts->../src/files/services/files-service.factory.ts + + + + + +integration/files.service.integration.spec.ts->../src/files/services/gitlab-files.service.ts + + + + + +integration/files.service.integration.spec.ts->../src/files/services/local-files.service.ts + + + + + +integration/files.service.integration.spec.ts->testUtil.ts + + + + + +unit/files-service.factory.unit.spec.ts + +unit/files-service.factory.unit.spec.ts + + + +unit/files-service.factory.unit.spec.ts->../src/files/services/files-service.factory.ts + + + + + +unit/files-service.factory.unit.spec.ts->../src/files/services/gitlab-files.service.ts + + + + + +unit/files-service.factory.unit.spec.ts->../src/files/services/local-files.service.ts + + + + + +unit/files-service.factory.unit.spec.ts->../src/files/interfaces/files.service.interface.ts + + + + + +unit/files.resolver.unit.spec.ts + +unit/files.resolver.unit.spec.ts + + + +unit/files.resolver.unit.spec.ts->../src/files/resolvers/files.resolver.ts + + + + + +unit/files.resolver.unit.spec.ts->../src/files/services/files-service.factory.ts + + + + + +unit/files.resolver.unit.spec.ts->../src/files/interfaces/files.service.interface.ts + + + + + +unit/files.resolver.unit.spec.ts->testUtil.ts + + + + + +unit/gitlab-files.service.unit.spec.ts + +unit/gitlab-files.service.unit.spec.ts + + + +unit/gitlab-files.service.unit.spec.ts->../src/files/services/gitlab-files.service.ts + + + + + +unit/gitlab-files.service.unit.spec.ts->testUtil.ts + + + + + +unit/local-files.service.unit.spec.ts + +unit/local-files.service.unit.spec.ts + + + +unit/local-files.service.unit.spec.ts->../src/files/services/local-files.service.ts + + + + + +unit/local-files.service.unit.spec.ts->testUtil.ts + + + + + diff --git a/docs/developer/system/C4-L1_diagram.png b/docs/developer/system/C4-L1_diagram.png new file mode 100644 index 000000000..7191fd342 Binary files /dev/null and b/docs/developer/system/C4-L1_diagram.png differ diff --git a/docs/developer/system/C4-L2_diagram_detailed.png b/docs/developer/system/C4-L2_diagram_detailed.png new file mode 100644 index 000000000..167a544c1 Binary files /dev/null and b/docs/developer/system/C4-L2_diagram_detailed.png differ diff --git a/docs/developer/system/C4-L2_diagram_simplified.png b/docs/developer/system/C4-L2_diagram_simplified.png new file mode 100644 index 000000000..7c08ea9d4 Binary files /dev/null and b/docs/developer/system/C4-L2_diagram_simplified.png differ diff --git a/docs/developer/system/DTaaS.drawio b/docs/developer/system/DTaaS.drawio index 31b45f9bc..220503f54 100644 --- a/docs/developer/system/DTaaS.drawio +++ b/docs/developer/system/DTaaS.drawio @@ -1,9 +1,12 @@ - - - + + + + + + @@ -128,10 +131,10 @@ - + - + @@ -193,6 +196,9 @@ + + + @@ -433,11 +439,8 @@ - - - - + @@ -584,1160 +587,399 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - + + - - + + - - + + - - - - - + + - - + + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - + + - + - - + + - - - - - + + - - - - - + + - - + + - - - - - + + - - + + - - + + - - - - + + + + - - + + - - + + - - + + - - + + - - - - - - - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - + + - - + + - - - - - + + - - + + - - - - + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - + + - - - - - + + - - + + - - + + - - + + - + - - - - - - - - - - - - - - + + - + + - - - - - - - - - - - + + - - + + - - + + - - + + - - + + - - - - - - - - - + + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - - - - - + + - - - - - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - + + - - + + - - - - - - - - - - - + + - - + + - - - - - - - - - - - + + - - + + - - + + - + - - + + - - - - + - - + + - - - - + + + + + - - - - - - - - - - - - - - - - - - - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - + + - - + + - - - - + - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - + + - - - - - + + - + - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + + + + + - - - - - + + - - - - - + + - - - - + - - + + - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - - - - - - - - - - + + - - + + - - + + - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + @@ -1785,13 +1027,12 @@ - + - - + @@ -1810,7 +1051,7 @@ - + @@ -2077,1212 +1318,295 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -4416,191 +2740,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/developer/system/architecture.md b/docs/developer/system/architecture.md new file mode 100644 index 000000000..656cf83f2 --- /dev/null +++ b/docs/developer/system/architecture.md @@ -0,0 +1,118 @@ +# :european_castle: System Overview + +## User Requirements + +The DTaaS software platform users expect a single platform +to support the complete DT lifecycle. To be more precise, the platform users expect the following features: + +1. **Author** – create different assets of the DT on the +platform itself. This step requires use of some software +frameworks and tools whose sole purpose is to author +DT assets. +1. **Consolidate** – consolidate the list of available DT assets +and authoring tools so that user can navigate the library +of reusable assets. This functionality requires support +for discovery of available assets. +1. **Configure** – support selection and configuration of +DTs. This functionality also requires support for validation of a given configuration. +1. **Execute** – provision computing infrastructure on demand to support execution of a DT. +1. **Explore** – interact with a DT and explore the results +stored both inside and outside the platform. Exploration +may lead to analytical insights. +1. **Save** – save the state of a DT that’s already in the +execution phase. This functionality is required for ondemand saving and re-spawning of DTs. +1. **What-if analysis** – explore alternative scenarios to (i) +plan for an optimal next step, (ii) recalibrate new DT +assets, (iii) automated creation of new DTs or their +assets; these newly created DT assets may be used to +perform scientifically valid experiments. +1. **Share** – share a DT with other users of their organisation. + +## System Architecture + +The figure shows the system architecture of the the DTaaS software platform. + +![System architecture](architecture.png) + +### System Components + +The users interact with the software platform using a website. +The gateway is a single point of entry for direct access to the platform +services. The gateway is responsible for controlling user access to +the microservice components. The service mesh +enables discovery of microservices, load balancing and authentication +functionalities. + +In addition, there are microservices for catering to author, store, +explore, configure, execute and scenario analysis requirements. +The microservices are complementary and composable; they fulfil +core requirements of the system. + +The microservices responsible for satisfying the user requirements are: + +1. **The security microservice** implements +role-based access control (RBAC) in the platform. +1. **The accounting microservice** is responsible for keeping track of the +platform, DT asset and infrastructure usage. Any licensing, +usage restrictions need to be enforced by the accounting +microservice. Accounting is a pre-requisite to commercialisation of +the platform. + + Due to significant use of external +infrastructure and resources via the platform, the accounting +microservice needs to interface with accounting systems of +the external services. + +1. **The data microservice** is a frontend to all the databases +integrated into the platform. A time-series database and a +graph database are essential. These two databases store timeseries +data from PT, events on PT/DT, commands sent by +DT to PT. The PTs uses these databases even when their +respective DTs are not in the execute phase. +1. **The visualisation microservice** is again a frontend to +visualisation software that are natively supported inside the platform. +Any visualisation software running either on external +systems or on client browsers do not need to interact with +this microservice. They can directly use the data provided by +the data microservice. + +## C4 Architectural Diagrams + +The C4 architectural diagrams of the DTaaS software are presented here. + +### Level 1 + +This Level 1 diagram only shows the users and the roles +they play in the DTaaS software. + +C4 Level 1 diagram + +### Level 2 + +This simplified version of Level 2 diagram shows +the software containers of the DTaaS software. + +![C4 Level 2 diagram](C4-L2_diagram_simplified.png) + +If you are interested, please take a look at +the [detailed diagram](C4-L2_diagram_detailed.png). + +Please note that the given diagram only +covers DT Lifecycle, Reusable Assets and Execution Manager. + +## Mapping + +A mapping of the C4 level 2 containers to components +identified in the system architecture is also available in the table. + +| System Component | Container(s) | +|:---|:---| +| Gateway | [Traefik Gateway](https://github.com/INTO-CPS-Association/DTaaS/tree/feature/distributed-demo/servers/config/gateway#the-gateway-server) | +| Unified Interface | [React Webapplication](./client.md) | +| Reusable Assets | [Library Microservice](./lib-ms.md) | +| Data | MQTT, InfluxDB, and RabbitMQ (not shown in the C4 Level 2 diagram) | +| Visualization | InfluxDB (not shown in the C4 Level 2 diagram) | +| DT Lifecycle | DT Lifecycle Manager and DT Configuration Validator | +| Security | [Gitlab OAuth](../../admin/client/auth.md) | +| Accounting | None | +| Execution Manager | Execution Manager | diff --git a/docs/developer/system/architecture.png b/docs/developer/system/architecture.png new file mode 100644 index 000000000..49151cd88 Binary files /dev/null and b/docs/developer/system/architecture.png differ diff --git a/docs/developer/system/current-status.md b/docs/developer/system/current-status.md new file mode 100644 index 000000000..2b203a321 --- /dev/null +++ b/docs/developer/system/current-status.md @@ -0,0 +1,91 @@ +# :clipboard: Current Status + +The DTaaS software platform is currently under development. +Crucial system components are in place with ongoing development work +focusing on increased automation and feature enhancement. +The figure below shows the current status of the development work. + +![Current development status](current-status.png) + +## :lock: User Security + +There is authentication mechanisms in place for the react website +and the Traefik gateway. + +The react website component uses Gitlab for user authentication using +OAuth protocol. + +### Gateway Authentication + +The Traefik gateway has HTTP basic authentication enabled by default. This +authentication on top of HTTPS connection can provide a good protection +against unauthorized use. + +!!! warning + Please note that HTTP basic authentication over insecure non-TLS + is insecure. + +There is also a possibility of using self-signed mTLS certificates. +The current security functionality is based on signed +Transport Layer Security (TLS) +certificates issued to users. The TLS certificate based mutual TLS (mTLS) +authentication protocol provides better security than the usual +username and password combination. The mTLS authentication takes place between +the users browser and the platform gateway. The gateway federates all +the backend services. The service discovery, load balancing, and health checks +are carried by the gateway based on a dynamic reconfiguration mechanism. + +!!! note + The mTLS is not enabled in the default install. + Please use the scripts in `ssl/` directory to generate + the required certificates for users and Traefik gateway. + +## :technologist: User Workspaces + +All users have dedicated dockerized-workspaces. These docker-images are based on +container images published by +[mltooling group](https://github.com/ml-tooling/ml-workspace). + +Thus DT experts can develop DTs from existing DT components and +share them with other users. A file server has been setup to act as +a DT asset repository. Each user gets space to store private DT assets and +also gets access to shared DT assets. Users can synchronize their +private DT assets with external git repositories. In addition, +the asset repository transparently gets mapped to user workspaces +within which users can perform DT lifecycle operations. +There is also a [library microservice](../servers/lib/lib-ms.md) which +in the long-run will replace the file server. + +Users can run DTs in their workspaces and also permit remote access +to other users. There is already shared access to internal and +external services. +With these two provisions, users can treat live DTs as service components +in their own software systems. + +## :electric_plug: Platform Services + +There are four external services integrated with the DTaaS software platform. +They are: +[InfluxDB](https://github.com/influxdata/influxdb), +[Grafana](https://github.com/grafana/grafana), +[RabbitMQ](https://github.com/rabbitmq/rabbitmq-server) +and +[MQTT](https://mqtt.org/). + +These services can be used by DTs and PTs for communication, storing and +visualization of data. There can also be monitoring services setup +based on these services. + +## Development Priorities + +The development priorities for the DTaaS software development team are: + +* [DT Runner](https://github.com/INTO-CPS-Association/DTaaS/tree/feature/distributed-demo/servers/execution/runner) +(API Interface to DT) +* Multi-user and microservice security +* Increased automation of installation procedures +* DT Configuration DSL ín the form of YAML schema +* UI for DT creation +* DT examples + +Your contributions and collaboration are highly welcome. diff --git a/docs/developer/system/current-status.png b/docs/developer/system/current-status.png new file mode 100644 index 000000000..737ec369c Binary files /dev/null and b/docs/developer/system/current-status.png differ diff --git a/docs/developer/testing/intro.md b/docs/developer/testing/intro.md new file mode 100644 index 000000000..dc13703d3 --- /dev/null +++ b/docs/developer/testing/intro.md @@ -0,0 +1,111 @@ +# :eye: Testing + +## :question: Common Questions on Testing + +### What is Software Testing? + +Software testing is a procedure to investigate the quality of a software product +in different scenarios. It can also be stated as the process of verifying and +validating that a software program or application works as expected and meets +the business and technical requirements that guided design and development. + +### Why Software Testing? + +Software testing is required to point out the defects and errors that were made +during different development phases. Software testing also ensures that +the product under test works as expected in all different cases – stronger +the test suite, stronger is our confidence in the product that we have built. +One important benefit of software testing is that it facilitates the developers +to make incremental changes to source code and make sure that the current +changes are not breaking the functionality of the previously existing code. + +### What is TDD? + +TDD stands for **Test Driven Development**. It is a software development process +that relies on the repetition of a very short development cycle: first +the developer writes an (initially failing) automated test case that defines +a desired improvement or new function, then produces the minimum amount of code +to pass that test, and finally refactors the new code to acceptable standards. +The goal of TDD can be viewed as specification and not validation. +In other words, it’s one way to think through your requirements or design +before your write your functional code. + +### What is BDD? + +BDD stands for “Behaviour Driven Development”. It is a software development +process that emerged from TDD. It includes the practice of writing tests first, +but focuses on tests which describe behavior, rather than tests which test +a unit of implementation. This provides software development and management +teams with shared tools and a shared process to collaborate on +software development. BDD is largely facilitated through the use of a simple +domain-specific language (DSL) using natural language constructs +(e.g., English-like sentences) that can express the behavior and the expected +outcomes. Mocha and Cucumber testing libraries are built around +the concepts of BDD. + +## :building_construction: Testing workflow + +We follow a testing workflow in accordance with +[The Test Pyramid](https://martinfowler.com/articles/practical-test-pyramid.html#TheTestPyramid), +starting with isolated tests and moving towards complete integration for +any new feature changes. The different types of tests (in the order that they +should be performed) are explained below: + +### [Unit Tests](https://martinfowler.com/articles/practical-test-pyramid.html#UnitTests) + +Unit testing is a level of software testing where individual units/ components +of a software are tested. The objective of Unit Testing is to isolate +a section of code and verify its correctness. + +Ideally, each test case is independent from the others. Substitutes such as +method stubs, mock objects, and spies can be used to assist testing a module in isolation. + +#### Benefits of Unit Testing + +* Unit testing increases confidence in changing/ maintaining code. +If good unit tests are written and if they are run every time any code is changed, +we will be able to promptly catch any defects introduced due to the change. +* If codes are already made less interdependent to make unit testing possible, +the unintended impact of changes to any code is less. +* The cost, in terms of time, effort and money, of fixing a defect detected during +unit testing is lesser in comparison to that of defects detected at higher levels. + +#### Unit Tests in DTaaS + +Each component DTaaS project uses unique technology stack. Thus the packages +used for unit tests are different. Please check the `test/` directory of +a component to figure out the unit test packages used. + +### [Integration tests](https://martinfowler.com/articles/practical-test-pyramid.html#IntegrationTests) + +Integration testing is the phase in software testing in which individual +software modules are combined and tested as a group. In DTaaS, we use +an [integration server](https://github.com/INTO-CPS-Association/DTaaS/wiki/DTaaS-Integration-Server) +for software development as well as such tests. + +The existing integration tests are done at the component level. +There are no integration tests between the components. +This task has been postponed to future. + +### [End-to-End tests](https://martinfowler.com/articles/practical-test-pyramid.html#End-to-endTests) + +Testing any code changes through the end user interface of your software +is essential to verify if your code has the desired effect for the user. +[End-to-End tests in DTaaS](https://github.com/INTO-CPS-Association/DTaaS/blob/feature/distributed-demo/client/test/README.md) +a functional setup. For more information +[visit here](https://github.com/INTO-CPS-Association/DTaaS/blob/feature/distributed-demo/client/test/README.md). + +There are end-to-end tests in the DTaaS. This task has been postponed to future. + +### Feature Tests + +A Software feature can be defined as the changes made in the system to add +new functionality or modify the existing functionality. Each feature is said to have +a characteristics that is designed to be useful, intuitive and effective. +It is important to test a new feature when it has been added. We also need to +make sure that it does not break the functionality of already existing features. +Hence feature tests prove to be useful. + +The DTaaS project does not have any feature tests yet. +[Cucumber](https://github.com/cucumber/cucumber-js) shall be used +in future to implement feature tests. diff --git a/docs/index.md b/docs/index.md index 81cfa941f..fa9d9775d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,18 +1,20 @@ -The Digital Twin as a Service (DTaaS) software is useful -to create and run digital twins. -The digital twins that are running can be used as service by other users. -These users need not be members of the DTaaS software platform itself. +# :factory: :left_right_arrow: :busts_in_silhouette: What is DTaaS -There is an overview of the software available for: +The Digital Twin as a Service (DTaaS) software platform is useful +to **Build, Use and Share** digital twins (DTs). -* General users : [overview slides](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/DTaaS-short-intro.pdf) - and [overview video](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/DTaaS-short-intro.mp4), - [feature walkthrough](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/dtaas-v0.2.0-demo.mp4) -* Developers : [slides](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/DTaaS-overview.pdf) - and [video](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/DTaaS-overview.mkv). +:mechanical_arm: **Build**: The DTs are built on the software platform +using the reusable DT components available on the platform. -There is also a [research paper draft](https://arxiv.org/abs/2305.07244) -if you are interested in reading the scientific roadmap for this software. +:office_worker: :factory_worker: **Use**: Use the DTs on the software platform. + +:handshake: **Share**: Share ready to use DTs with other users. +It is also possible to share the services offered by one DT with other users. + +There is an overview of the software available in the form of +[slides](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/DTaaS-short-intro.pdf), +[video](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/DTaaS-short-intro.mp4), +and [feature walkthrough](https://odin.cps.digit.au.dk/into-cps/dtaas/assets/videos/dtaas-v0.2.0-demo.mp4). ## License diff --git a/docs/user/current-status.png b/docs/user/current-status.png index 15655c941..dd9272aab 100755 Binary files a/docs/user/current-status.png and b/docs/user/current-status.png differ diff --git a/docs/user/digital-twins/build-use-share.png b/docs/user/digital-twins/build-use-share.png new file mode 100755 index 000000000..db55aa292 Binary files /dev/null and b/docs/user/digital-twins/build-use-share.png differ diff --git a/docs/user/digital-twins/create.md b/docs/user/digital-twins/create.md index 9da34cd18..033cfafd0 100644 --- a/docs/user/digital-twins/create.md +++ b/docs/user/digital-twins/create.md @@ -1,33 +1,58 @@ # Create a Digital Twin -The first step in digital twin creation is to use the available assets in your workspace. If you have assets / files in your computer that need to be available in the DTaaS workspace, then please follow the instructions provided in [library assets](../servers/lib/assets.md). +The first step in digital twin creation is to use the available assets +in your workspace. If you have assets / files in your computer that need +to be available in the DTaaS workspace, then please follow the instructions +provided in [library assets](../servers/lib/assets.md). -There are dependencies among the library assets. These dependencies are shown below. +There are dependencies among the library assets. These dependencies +are shown below. ![Relation between reusable assets](asset-relationship.png) -A digital twin can only be created by linking the assets in a meaningful way. This relationship can be expressed using a mathematical equation: +A digital twin can only be created by linking the assets in +a meaningful way. This relationship can be expressed using +a mathematical equation: $$ D_t: \{ D{^*},M^{*},(FT)^{+} \}C_{dt} $$ -where D denotes data, M denotes models, F denotes functions, T denotes tools, $C_{dt}$ denotes DT configuration and $D_t$ is a symbolic notation for a digital twin itself. The $\{ D{^*},M^{*},(FT)^{+} \}C_{dt}$ expression denotes composition of DT from D,M,T and F assets. The $*$ indicates zero or one more instances of an asset and $+$ indicates one or more instances of an asset. +where D denotes data, M denotes models, F denotes functions, T denotes tools, +$C_{dt}$ denotes DT configuration and $D_t$ is a symbolic notation for +a digital twin itself. The $\{ D{^*},M^{*},(FT)^{+} \}C_{dt}$ expression +denotes composition of DT from D,M,T and F assets. The $*$ indicates zero +or one more instances of an asset and $+$ indicates one or more instances +of an asset. -The DT configuration specifies the relevant assets to use, the potential parameters to be set for these assets. If a DT needs to use RabbitMQ, InfluxDB like services supported by the platform, the DT configuration needs to have access credentials for these services. +The DT configuration specifies the relevant assets to use, the potential +parameters to be set for these assets. If a DT needs to use RabbitMQ, InfluxDB +like services supported by the platform, the DT configuration needs to +have access credentials for these services. -This kind of generic DT definition is based on the DT examples seen in the wild. You are at liberty to deviate from this definition of DT. The only requirement is the ability to run the DT from either commandline or desktop. +This kind of generic DT definition is based on the DT examples seen in +the wild. You are at liberty to deviate from this definition of DT. +The only requirement is the ability to run the DT from either commandline +or desktop. !!! tip - If you are stepping into the world of Digital Twins, you might not have distinct digital twin assets. You are likely to have one directory of everything in which you run your digital twin. In such a case we recommend that you upload this monolithic digital twin into **digital twin/your_digital_twin_name** directory. + If you are stepping into the world of Digital Twins, you might not + have distinct digital twin assets. You are likely to have one directory + of everything in which you run your digital twin. In such a case we + recommend that you upload this monolithic digital twin into + **digital twin/your_digital_twin_name** directory. ## Example -The [Examples](https://github.com/INTO-CPS-Association/DTaaS-examples) repository contains a co-simulation setup for mass spring damper. The complete details on this example are available on [github](https://github.com/INTO-CPS-Association/example-mass_spring_damper). +The [Examples](https://github.com/INTO-CPS-Association/DTaaS-examples) +repository contains a co-simulation setup for mass spring damper. +The complete details on this example are available on +[github](https://github.com/INTO-CPS-Association/example-mass_spring_damper). -This example illustrates the potential of using co-simulation for digital twins. +This example illustrates the potential of using co-simulation +for digital twins. The file system contents for this example are: @@ -66,37 +91,68 @@ workspace/ maestro-2.3.0-jar-with-dependencies.jar ``` -The `workspace/data/mass-spring-damper/` contains `input` and `output` data for the mass-spring-damper digital twin. +The `workspace/data/mass-spring-damper/` contains `input` and +`output` data for the mass-spring-damper digital twin. -The two FMU models needed for this digital twin are in `models/` directory. +The two FMU models needed for this digital twin are in +`models/` directory. -The co-simulation digital twin needs Maestro co-simulation orchestrator. Since this is a reusable asset for all the co-simulation based DTs, the tool has been placed in `common/tools/` directory. +The co-simulation digital twin needs Maestro co-simulation +orchestrator. Since this is a reusable asset for all +the co-simulation based DTs, the tool has been placed in +`common/tools/` directory. -The actual digital twin configuration is specified in `digital twins/mass-spring-damper` directory. The co-simulation configuration is specified in two json files, namely `cosim.json` and `time.json`. A small explanation of digital twin for its users can be placed in `digital twins/mass-spring-damper/README.md`. +The actual digital twin configuration is specified in +`digital twins/mass-spring-damper` directory. The co-simulation configuration +is specified in two json files, namely `cosim.json` and `time.json`. +A small explanation of digital twin for its users can be placed in +`digital twins/mass-spring-damper/README.md`. -The launch program for this digital twin is in `digital twins/mass-spring-damper/lifecycle/execute`. This launch program runs the co-simulation digital twin. The co-simulation runs till completion and then ends. The programs in `digital twins/mass-spring-damper/lifecycle` are responsible for lifecycle management of this digital twin. The [lifecycle page](lifecycle.md) provides more explanation on these programs. +The launch program for this digital twin is in +`digital twins/mass-spring-damper/lifecycle/execute`. This launch program runs +the co-simulation digital twin. The co-simulation runs till completion and +then ends. The programs in `digital twins/mass-spring-damper/lifecycle` are +responsible for lifecycle management of this digital twin. +The [lifecycle page](lifecycle.md) provides more explanation on these programs. !!! Abstract "Execution of a Digital Twin" - A frequent question arises on the run time characteristics of a digital twin. The natural intuition is to say that a digital twin must operate as long as its physical twin is in operation. - **If a digital twin runs for a finite time and then ends, can it be called a digital twin?** + A frequent question arises on the run time characteristics of + a digital twin. The natural intuition is to say that a digital twin must + operate as long as its physical twin is in operation. - **The answer is a resounding YES**. The Industry 4.0 usecases seen among SMEs have digital twins that run for a finite time. These digital twins are often run at the discretion of the user. + **If a digital twin runs for a finite time and then ends, can it be + called a digital twin?** -**You can run this digital twin by ** + **The answer is a resounding YES**. The Industry 4.0 usecases seen among + SMEs have digital twins that run for a finite time. These digital twins + are often run at the discretion of the user. -1. Go to Workbench tools page of the DTaaS website and open VNC Desktop. This opens a new tab in your browser -2. A page with VNC Desktop and a connect button comes up. Click on Connect. You are now connected to the Linux Desktop of your workspace. -3. Open a Terminal (black rectangular icon in the top left region of your tab) and type the following commands. -4. Download the [example files](https://github.com/INTO-CPS-Association/DTaaS-examples/archive/refs/heads/main.zip) - ``` +**You can run this digital twin by**, + +1. Go to Workbench tools page of the DTaaS website and open VNC Desktop. +This opens a new tab in your browser +1. A page with VNC Desktop and a connect button comes up. Click on Connect. +You are now connected to the Linux Desktop of your workspace. +1. Open a Terminal (black rectangular icon in the top left region of your tab) +and type the following commands. +1. Download the +[example files](https://github.com/INTO-CPS-Association/DTaaS-examples/archive/refs/heads/main.zip) + + ```sh $wget https://github.com/INTO-CPS-Association/DTaaS-examples/archive/refs/heads/main.zip $unzip main.zip ``` -5. Open a file browser and copy the files from this uncompressed folder into your workspace folder (`/workspace`). Make sure that the file placement matches the one given above. -6. Go to the digital twin directory and run + +1. Open a file browser and copy the files from this uncompressed folder +into your workspace folder (`/workspace`). Make sure that the file placement +matches the one given above. +1. Go to the digital twin directory and run + ``` $cd /workspace/digital twins/mass-spring-damper $lifecycle/execute ``` - The last command executes the mass-spring-damper digital twin and stores the co-simulation output in `data/mass-spring-damper/output`. \ No newline at end of file + + The last command executes the mass-spring-damper digital twin and stores + the co-simulation output in `data/mass-spring-damper/output`. diff --git a/docs/user/digital-twins/lifecycle-four-stages.png b/docs/user/digital-twins/lifecycle-four-stages.png new file mode 100755 index 000000000..f55f1b2c7 Binary files /dev/null and b/docs/user/digital-twins/lifecycle-four-stages.png differ diff --git a/docs/user/digital-twins/lifecycle.md b/docs/user/digital-twins/lifecycle.md index d7238356f..0bf0e8d98 100644 --- a/docs/user/digital-twins/lifecycle.md +++ b/docs/user/digital-twins/lifecycle.md @@ -1,7 +1,25 @@ +# :recycle: Digital Twin Lifecycle -![Digital Twin Lifecycle](lifecycle.png) +The physical products in the real world have product lifecycle. +A simplified four-stage product life is illustrated here. + +A digital twin tracking the physical products (twins) need +to track and evolve in conjunction with the corresponding +physical twin. + +The possible activities undertaken in each lifecycle phases +are illustrated in the figure. + +![DT-PT Lifecycle](lifecycle-four-stages.png) + +## Lifecycle Phases + +The four phase lifecycle has been extended to a lifecycle with +eight phases. The new phase names and the typical activities +undertaken in each phase are outlined in this section. -A DT lifecycle consists of **explore, create, execute, save, analyse, evolve** and **terminate** phases. + A DT lifecycle consists of **explore, create, execute, save, analyse, evolve** + and **terminate** phases. | Phase | Main Activities | |:----|:----| @@ -13,9 +31,28 @@ A DT lifecycle consists of **explore, create, execute, save, analyse, evolve** a | **save** | involves saving the state of DT to enable future recovery. | | **terminate** | stop the execution of DT. | -A complete digital twin will support all the phases but it is not mandatory. +A digital twin faithfully tracking the physical twin lifecycle will have to +support all the phases. It is also possible for digital twin engineers to add +more phases to digital they are developing. Thus it is important for +the DTaaS software platform needs to accommodate needs of different DTs. + +A potential linear representation of the tasks undertaken in +a digital twin lifecycle are shown here. + +![Digital Twin Lifecycle](lifecycle.png) + +Again this is only a one possible pathway. Users are at liberty to +alter the sequence of steps. + +It is possible to map the lifecycle phases identified so far with +the **Build-Use-Share** approach +of the DTaaS software platform. + +![DT Lifecycle and Build-Use-Share](build-use-share.png) -Even though not mandatory, having a coding structure makes it easy to manage DT lifecycle phases. It is recommended to have the following structure +Even though not mandatory, having a matching coding structure makes it easy to +for users to create and manage their DTs within the DTaaS. +It is recommended to have the following structure: ```text workspace/ @@ -30,11 +67,14 @@ workspace/ terminate ``` -A dedicated program exists for each phase of DT lifecycle. Each program can be as simple as a script that launches other programs or sends messages to a live digital twin. +A dedicated program exists for each phase of DT lifecycle. Each program +can be as simple as a script that launches other programs or sends messages +to a live digital twin. ## Examples -Here are the programs / scripts to manage three phases in the lifecycle of **mass-spring-damper DT**. +Here are the programs / scripts to manage three phases in +the lifecycle of **mass-spring-damper DT**. ```bash title="lifecycle/execute" #!/bin/bash @@ -46,16 +86,21 @@ java -jar /workspace/common/tools/maestro-2.3.0-jar-with-dependencies.jar \ output-dir>debug.log 2>&1 ``` -The execute phases uses the DT configuration, FMU models and Maestro tool to execute the digital twin. The script also stores the output of cosimulation in `/workspace/data/mass-spring-damper/output`. +The execute phases uses the DT configuration, FMU models and Maestro tool +to execute the digital twin. The script also stores the output of +cosimulation in `/workspace/data/mass-spring-damper/output`. -It is possible for a DT not to support a specific lifecycle phase. This intention can be specified with an empty script and a helpful message if deemed necessary. +It is possible for a DT not to support a specific lifecycle phase. +This intention can be specified with an empty script and a helpful message +if deemed necessary. ```bash title="lifecycle/analyze" #!/bin/bash printf "operation is not supported on this digital twin" ``` -The lifecycle programs can call other programs in the code base. In the case of `lifecycle/terminate` program, it is calling another script to do the necessary job. +The lifecycle programs can call other programs in the code base. In the case of +`lifecycle/terminate` program, it is calling another script to do the necessary job. ```bash title="lifecycle/terminate" #!/bin/bash diff --git a/docs/user/digital-twins/lifecycle.png b/docs/user/digital-twins/lifecycle.png index ce6c7b3b7..583fa2ddd 100755 Binary files a/docs/user/digital-twins/lifecycle.png and b/docs/user/digital-twins/lifecycle.png differ diff --git a/docs/user/features.md b/docs/user/features.md index 096f0152a..94eb3ebb6 100644 --- a/docs/user/features.md +++ b/docs/user/features.md @@ -1,3 +1,18 @@ +# Overview + +## Advantages + +The DTaaS software platform provides certain advantages to users: + +* Support for different kinds of Digital Twins - CFD, Simulink, co-simulation, FEM, ROM, ML etc. +* Integrates with other Digital Twin frameworks +* Facilitate availability of Digital Twin as a service +* Collaboration and reuse +* Private workspaces for authoring and verification of reusable assets, trial run DTs +* Cost effectiveness + +## Software Features + Each installation of DTaaS platform comes with the features highlighted in the following picture. ![Features](current-status.png) @@ -20,9 +35,10 @@ The DTaaS software platform has some pre-installed services available. The curre | Service | Advantage | |:---|:---| -| InfluxDB | time-series database primarly for storing time-series data from physical twins. The digital twins can use an already existing data. Users can also create visualization dashboards for their digital twins. | -| RabbitMQ | communication broker for communication between physical and digital twins | -| Grafana | Users can create visualization dashboards for their digital twins. | +| InfluxDB | Time-series database primarly for storing time-series data from physical twins. The digital twins can use an already existing data. Users can also create visualization dashboards for their digital twins. | +| RabbitMQ | Communication broker for communication between physical and digital twins | +| Grafana | Visualization dashboards for their digital twins. | +| MQTT | Lightweight data transfer broker for IoT devices / physical twins feeding data into digital twins. | In addition, the workspaces are connected to the Internet so all the Digital Twins run within their workspace can interact with both the internal and external services. diff --git a/docs/user/motivation.md b/docs/user/motivation.md new file mode 100644 index 000000000..8f215e0bc --- /dev/null +++ b/docs/user/motivation.md @@ -0,0 +1,34 @@ +# Motivation + +How can DT software platforms enable users collaborate to: + +* Build digital twins (DTs) +* Use DTs themselves +* Share DTs with other users +* Provide the existing DTs as Service to other users + +In addition, how can the DT software platforms: + +* Support DT lifecycle +* Scale up rather than scale down (flexible convention over configuration) + +## Existing Approaches + +There are quite a few solutions proposed in the recent past to solve +this problem. Some of them are: + +* Focus on data from Physical Twins (PTs) to perform analysis, diagnosis, planning etc… +* Share DT assets across the upstream, downstream etc…. +* Evaluate different models of PT +* DevOps for Cyber Physical Systems (CPS) +* Scale DT / execution of DT / ensemble of related DTs +* Support for PT product lifecycle + +## Our Approach + +* Support for transition from existing workflows to DT frameworks +* Create DTs from reusable assets +* Enable users to share DT assets +* Offer DTs as a service +* Integrate the DTs with external software systems +* Separate configurations of independent DT components diff --git a/docs/user/website/index.md b/docs/user/website/index.md index 910b3d603..a31f93d96 100644 --- a/docs/user/website/index.md +++ b/docs/user/website/index.md @@ -1,61 +1,107 @@ -This page contains a screenshot driven preview of the website. +# DTaaS Website Screenshots -### Login to enter the DTaaS software platform +This page contains a screenshot driven preview of the website serving +the DTaaS software platform. + +## Login to enter the DTaaS software platform ![Login](login.png) -The screen presents with HTTP authentication form. You can enter the user credentials. You will be using HTTPS secure communication so the username and password are secure. -### Enter username again +The screen presents with HTTP authentication form. You can enter the user +credentials. You will be using HTTPS secure communication so the username +and password are secure. + +## Enter username again ![Username](username.png) -You are now logged into the server. You can enter the same username again to log into your workspace. +You are now logged into the server. You can enter the same username again +to log into your workspace. -### Overview of menu items +## Overview of menu items -The menu is hidden by default. Only the icons of menu items are visible. You can click on the :octicons-three-bars-16: icon in the top-left corner of the page to see the menu. +The menu is hidden by default. Only the icons of menu items are visible. +You can click on the :octicons-three-bars-16: icon in the top-left corner +of the page to see the menu. ![Menu](menu.png) There are three menu items: -**Library**: for management of reusable library assets. You can upload, download, create and modify new files on this page. +**Library**: for management of reusable library assets. You can upload, +download, create and modify new files on this page. -**Digital Twins**: for management of digital twins. You are presented with Jupyter Lab page from which you can run the digital twins. +**Digital Twins**: for management of digital twins. You are presented with +the Jupyter Lab page from which you can run the digital twins. -**Workbench**: Not all digital twins can be managed within Jupyter Lab. You have more tools at your disposal on this page. +**Workbench**: Not all digital twins can be managed within Jupyter Lab. +You have more tools at your disposal on this page. - -### Library tabs and their help text +## Library tabs and their help text ![Menu](library.png) -You can see the file manager and five tabs above the library manager. Each tab provides help text to guide users in the use of different directories in their workspace. +You can see the file manager and five tabs above the library manager. Each tab +provides help text to guide users in the use of different directories +in their workspace. -??? Functions tip - The functions responsible for pre- and post-processing of: data inputs, data outputs, control outputs. The data science libraries and functions can be used to create useful function assets for the platform. +??? Functions tip + The functions responsible for pre- and post-processing of: data inputs, + data outputs, control outputs. The data science libraries and functions + can be used to create useful function assets for the platform. - In some cases, Digital Twin models require calibration prior to their use; functions written by domain experts along with right data inputs can make model calibration an achievable goal. Another use of functions is to process the sensor and actuator data of both Physical Twins and Digital Twins. + In some cases, Digital Twin models require calibration prior to their use; + functions written by domain experts along with right data inputs can make + model calibration an achievable goal. Another use of functions is to process + the sensor and actuator data of both Physical Twins and Digital Twins. ``` ??? Data tip - The data sources and sinks available to a digital twins. Typical examples of data sources are sensor measurements from Physical Twins, and test data provided by manufacturers for calibration of models. Typical examples of data sinks are visualization software, external users and data storage services. There exist special outputs such as events, and commands which are akin to control outputs from a Digital Twin. These control outputs usually go to Physical Twins, but they can also go to another Digital Twin. + The data sources and sinks available to a digital twins. Typical examples + of data sources are sensor measurements from Physical Twins, and + test data provided by manufacturers for calibration of models. + + Typical examples of data sinks are visualization software, external users + and data storage services. There exist special outputs such as events, and + commands which are akin to control outputs from a Digital Twin. + These control outputs usually go to Physical Twins, but they can also + go to another Digital Twin. ??? Models tip - The model assets are used to describe different aspects of Physical Twins and their environment, at different levels of abstraction. Therefore, it is possible to have multiple models for the same Physical Twin. For example, a flexible robot used in a car production plant may have structural model(s) which will be useful in tracking the wear and tear of parts. The same robot can have a behavioural model(s) describing the safety guarantees provided by the robot manufacturer. The same robot can also have a functional model(s) describing the part manufacturing capabilities of the robot. + The model assets are used to describe different aspects of Physical Twins + and their environment, at different levels of abstraction. Therefore, + it is possible to have multiple models for the same Physical Twin. + For example, a flexible robot used in a car production plant may have + structural model(s) which will be useful in tracking the wear and tear + of parts. The same robot can have a behavioural model(s) describing + the safety guarantees provided by the robot manufacturer. The same robot + can also have a functional model(s) describing the part manufacturing + capabilities of the robot. ??? Tools tip - The software tool assets are software used to create, evaluate and analyze models. These tools are executed on top of a computing platforms, i.e., an operating system, or virtual machines like Java virtual machine, or inside docker containers. The tools tend to be platform specific, making them less reusable than models. - - A tool can be packaged to run on a local or distributed virtual machine environments thus allowing selection of most suitable execution environment for a Digital Twin. - - Most models require tools to evaluate them in the context of data inputs. - There exist cases where executable packages are run as binaries in a computing environment. Each of these packages are a pre-packaged combination of models and tools put together to create a ready to use Digital Twins. + The software tool assets are software used to create, evaluate and + analyze models. These tools are executed on top of a computing + platforms, i.e., an operating system, or virtual machines like + Java virtual machine, or inside docker containers. The tools tend + to be platform specific, making them less reusable than models. + + A tool can be packaged to run on a local or distributed virtual machine + environments thus allowing selection of most suitable execution + environment for a Digital Twin. + + Most models require tools to evaluate them in the context of data inputs. + There exist cases where executable packages are run as binaries in + a computing environment. Each of these packages are a pre-packaged + combination of models and tools put together to create a ready to + use Digital Twins. ??? Digital Twins tip - These are ready to use digital twins created by one or more users. These digital twins can be reconfigured later for specific use cases. + These are ready to use digital twins created by one or more users. + These digital twins can be reconfigured later for specific use cases. -In addition to the five directories, there is also **common** directory in which five sub-directories exist. These sub-directories are: data, functions, models, tools and digital twins. +In addition to the five directories, there is also **common** directory +in which five sub-directories exist. These sub-directories are: +data, functions, models, tools and digital twins. ??? Common Assets tip The common directory again has four sub-directories: @@ -68,48 +114,73 @@ In addition to the five directories, there is also **common** directory in which The assets common to all users are placed in **common**. +The items used by more than one user are placed in **common**. The items in +the **common** directory are available to all users. Further explanation of +directory structure and placement of reusable assets within the the directory +structure is in the [assets page](../servers/lib/assets.md#file-system-structure) -The items used by more than one user are placed in **common**. The items in the **common** directory are available to all users. Further explanation of directory structure and placement of reusable assets within the the directory structure is in the [assets page](../servers/lib/assets.md#file-system-structure) +:fontawesome-solid-circle-info: The file manager is based on Jupyter notebook +and all the tasks you can perform in the Jupyter Notebook can be +undertaken here. -:fontawesome-solid-circle-info: The file manager is based on Jupyter Notebook and all the tasks you can perform in the Jupyter Notebook can be undertaken here. - -### Digital Twins page +## Digital Twins page ![Menu](digital_twins.png) -The digital twins page has three tabs and the central pane opens Jupyter Lab. There are three tabs with helpful instructions on the suggested tasks you can undertake in the **Create - Execute - Analyze** life cycle phases of digital twin. You can see more explanation on the [life cycle phases of digital twin](../digital-twins/lifecycle.md). +The digital twins page has three tabs and the central pane opens Jupyter lab. +There are three tabs with helpful instructions on the suggested tasks you can +undertake in the **Create - Execute - Analyze** life cycle phases of +digital twin. You can see more explanation on +the [life cycle phases of digital twin](../digital-twins/lifecycle.md). ??? Create tip - Create digital twins from tools provided within user workspaces. Each digital twin will have one directory. It is suggested that user provide one bash shell script to run their digital twin. Users can create the required scripts and other files from tools provided in Workbench page. + Create digital twins from tools provided within user workspaces. + Each digital twin will have one directory. It is suggested that user + provide one bash shell script to run their digital twin. Users can + create the required scripts and other files from tools provided in + Workbench page. ??? Execute tip - Digital twins are executed from within user workspaces. The given bash script gets executed from digital twin directory. Terminal-based digital twins can be executed from VSCode and graphical digital twins can be executed from VNC GUI. The results of execution can be placed in the data directory. + Digital twins are executed from within user workspaces. The given + bash script gets executed from digital twin directory. Terminal-based + digital twins can be executed from VSCode and graphical digital twins + can be executed from VNC GUI. The results of execution can be placed + in the data directory. ??? Analyze tip - The analysis of digital twins requires running of digital twin script from user workspace. The execution results placed within data directory are processed by analysis scripts and results are placed back in the data directory. These scripts can either be executed from VSCode and graphical results or can be executed from VNC GUI. + The analysis of digital twins requires running of digital twin script + from user workspace. The execution results placed within data directory + are processed by analysis scripts and results are placed back in + the data directory. These scripts can either be executed from VSCode + and graphical results or can be executed from VNC GUI. -:fontawesome-solid-circle-info: The reusable assets (files) seen in the file manager are available in the Jupyter Lab. In addition, there is a git plugin installed in the Jupyter Lab using which you can link your files with the external git repositories. +:fontawesome-solid-circle-info: The reusable assets (files) seen in +the file manager are available in the Jupyter Lab. In addition, there is +a git plugin installed in the Jupyter Lab using which you can link your +files with the external git repositories. -### Workbench +## Workbench The **workbench** page provides links to four integrated tools. ![Workbench](workbench.png) -The hyperlinks open in new browser tab. The screenshots of pages opened in new browser are: +The hyperlinks open in new browser tab. The screenshots of pages opened +in new browser are: ![Workbench Tools](workbench_tools.png) !!! Bug - The Terminal hyperlink does not always work reliably. If you want terminal. Please use the tools dropdown in the Jupyter Notebook. - + The Terminal hyperlink does not always work reliably. If you want + terminal. Please use the tools dropdown in the Jupyter Notebook. -### Finally logout +## Finally logout ![Logout](logout.png) -You have to close the browser in order to completely exit the DTaaS software platform. \ No newline at end of file +You have to close the browser in order to completely exit +the DTaaS software platform. diff --git a/docs/user/workflow.png b/docs/user/workflow.png new file mode 100644 index 000000000..c24f5b805 Binary files /dev/null and b/docs/user/workflow.png differ diff --git a/files/README.md b/files/README.md index 26cb32e33..c88ee780e 100644 --- a/files/README.md +++ b/files/README.md @@ -1 +1,5 @@ -This directory contains directory structure template for user files of the platform. These files are served by `servers/lib` microservice. +# User files + +This directory contains directory structure template for +storing the user files of the platform. These files are +mapped to `/workspace` directory in user workspaces. diff --git a/mkdocs-github.yml b/mkdocs-github.yml index 0224f1b3b..1ca477037 100644 --- a/mkdocs-github.yml +++ b/mkdocs-github.yml @@ -27,7 +27,9 @@ nav: - Separate Packages: - website: admin/client/CLIENT.md - library microservice: admin/servers/lib/LIB-MS.md + - Services: admin/services.md - User: + - Motivation: user/motivation.md - Features: user/features.md - Website: user/website/index.md - Library: @@ -38,6 +40,15 @@ nav: - Lifecycle: user/digital-twins/lifecycle.md - Examples: https://github.com/INTO-CPS-Association/DTaaS-examples - FAQ: FAQ.md + - Developer: + - Overview: developer/index.md + - System: + - Architecture: developer/system/architecture.md + - Current Status: developer/system/current-status.md + - Components: + - Client: developer/client/client.md + - Library Microservice: developer/servers/lib/lib-ms.md + - Testing: developer/testing/intro.md - Bugs: bugs.md - Thanks: thanks.md - License: LICENSE.md @@ -63,13 +74,22 @@ markdown_extensions: pygments_lang_class: true - pymdownx.inlinehilite - pymdownx.snippets - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.details - admonition - pymdownx.tabbed: alternate_style: true - mdx_math: enable_dollar_delimiter: True + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format plugins: - open-in-new-tab diff --git a/mkdocs.yml b/mkdocs.yml index 615674e7b..a70406af2 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,7 +27,9 @@ nav: - Separate Packages: - website: admin/client/CLIENT.md - library microservice: admin/servers/lib/LIB-MS.md + - Services: admin/services.md - User: + - Motivation: user/motivation.md - Features: user/features.md - Website: user/website/index.md - Library: @@ -38,6 +40,15 @@ nav: - Lifecycle: user/digital-twins/lifecycle.md - Examples: https://github.com/INTO-CPS-Association/DTaaS-examples - FAQ: FAQ.md + - Developer: + - Overview: developer/index.md + - System: + - Architecture: developer/system/architecture.md + - Current Status: developer/system/current-status.md + - Components: + - Client: developer/client/client.md + - Library Microservice: developer/servers/lib/lib-ms.md + - Testing: developer/testing/intro.md - Bugs: bugs.md - Thanks: thanks.md - License: LICENSE.md @@ -63,13 +74,22 @@ markdown_extensions: pygments_lang_class: true - pymdownx.inlinehilite - pymdownx.snippets - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.details - admonition - pymdownx.tabbed: alternate_style: true - mdx_math: enable_dollar_delimiter: True + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format plugins: - open-in-new-tab diff --git a/script/env.sh b/script/env.sh index 12185469e..8c853701a 100755 --- a/script/env.sh +++ b/script/env.sh @@ -84,4 +84,8 @@ sudo -H pip3 install mkdocs-with-pdf sudo -H pip3 install qrcode # Install markdownlint -sudo gem install mdl \ No newline at end of file +sudo gem install mdl + +# Install madge for generating dependency graphs of typescript projects +sudo apt-get install graphviz +sudo npm install -g madge \ No newline at end of file diff --git a/servers/execution/runner/.madgerc b/servers/execution/runner/.madgerc new file mode 100644 index 000000000..4425cb8ef --- /dev/null +++ b/servers/execution/runner/.madgerc @@ -0,0 +1,18 @@ +{ + "fontSize": "10px", + "tsconfig": "tsconfig.json", + "fileExtensions": ["ts", "tsx", "js", "jsx"], + "backgroundColor": "#FFFFFF", + "textColor": "#FFFFFF", + "nodeColor": "black", + "noDependencyColor": "green", + "cyclicNodeColor": "red", + "edgeColor": "green", + "graphVizOptions": { + "G": { + "rankdir": "TB", + "layout": "neato", + "splines": "curved" + } + } +} \ No newline at end of file diff --git a/servers/execution/runner/README.md b/servers/execution/runner/README.md index 723b2119f..4be89aa9e 100644 --- a/servers/execution/runner/README.md +++ b/servers/execution/runner/README.md @@ -20,6 +20,7 @@ One digital twin runner is responsible for execution of a digital twin. ```bash yarn install # Install dependencies for the microservice yarn syntax # analyzes source code for potential errors, style violations, and other issues, +yarn graph # generate dependency graphs in the code yarn build # compile ES6 files into ES5 javascript files and copy all JS files into build/ directory yarn test # run tests yarn test:nocov # run the tests but do not report coverage diff --git a/servers/execution/runner/package.json b/servers/execution/runner/package.json index df040d334..d753356d1 100644 --- a/servers/execution/runner/package.json +++ b/servers/execution/runner/package.json @@ -16,7 +16,8 @@ "syntax": "script/syntax.bash", "test": "script/test.bash", "test:nocov": "script/test.bash nocoverage", - "test:watchAll": "script/test.bash watchAll" + "test:watchAll": "script/test.bash watchAll", + "graph": "script/graph.bash" }, "bin": { "runner": "./dist/src/main.js" diff --git a/servers/execution/runner/script/clean.bash b/servers/execution/runner/script/clean.bash index d5c466b7d..e2b62928a 100755 --- a/servers/execution/runner/script/clean.bash +++ b/servers/execution/runner/script/clean.bash @@ -1,2 +1,3 @@ #!/bin/bash -rm -rf build node_modules coverage dist \ No newline at end of file +rm -rf build node_modules coverage dist +rm src.svg test.svg \ No newline at end of file diff --git a/servers/execution/runner/script/graph.bash b/servers/execution/runner/script/graph.bash new file mode 100755 index 000000000..57f177c8e --- /dev/null +++ b/servers/execution/runner/script/graph.bash @@ -0,0 +1,7 @@ +#!/bin/bash +PATH="$(yarn bin):$PATH" +export PATH +printf "Generate dependency graph for code" + +madge --image src.svg src +madge --image test.svg test diff --git a/servers/lib/.madgerc b/servers/lib/.madgerc new file mode 100644 index 000000000..4425cb8ef --- /dev/null +++ b/servers/lib/.madgerc @@ -0,0 +1,18 @@ +{ + "fontSize": "10px", + "tsconfig": "tsconfig.json", + "fileExtensions": ["ts", "tsx", "js", "jsx"], + "backgroundColor": "#FFFFFF", + "textColor": "#FFFFFF", + "nodeColor": "black", + "noDependencyColor": "green", + "cyclicNodeColor": "red", + "edgeColor": "green", + "graphVizOptions": { + "G": { + "rankdir": "TB", + "layout": "neato", + "splines": "curved" + } + } +} \ No newline at end of file diff --git a/servers/lib/README.md b/servers/lib/README.md index e9ba81fdd..5080e8b94 100644 --- a/servers/lib/README.md +++ b/servers/lib/README.md @@ -68,6 +68,7 @@ nohup yarn start & disown ```bash yarn install # Install dependencies for the microservice yarn syntax # analyzes source code for potential errors, style violations, and other issues, +yarn graph # generate dependency graphs in the code yarn build # compile ES6 files into ES5 javascript files and copy all JS files into build/ directory yarn test -a # run all tests yarn test -e # run end-to-end tests diff --git a/servers/lib/package.json b/servers/lib/package.json index 1d1d70972..2592e3e41 100644 --- a/servers/lib/package.json +++ b/servers/lib/package.json @@ -13,7 +13,8 @@ "clean": "script/clean.bash", "start": "script/start.bash", "syntax": "script/syntax.bash", - "test": "script/test.bash" + "test": "script/test.bash", + "graph": "script/graph.bash" }, "dependencies": { "@apollo/client": "^3.7.10", diff --git a/servers/lib/script/clean.bash b/servers/lib/script/clean.bash index d5c466b7d..e2b62928a 100755 --- a/servers/lib/script/clean.bash +++ b/servers/lib/script/clean.bash @@ -1,2 +1,3 @@ #!/bin/bash -rm -rf build node_modules coverage dist \ No newline at end of file +rm -rf build node_modules coverage dist +rm src.svg test.svg \ No newline at end of file diff --git a/servers/lib/script/graph.bash b/servers/lib/script/graph.bash new file mode 100755 index 000000000..4546da092 --- /dev/null +++ b/servers/lib/script/graph.bash @@ -0,0 +1,7 @@ +#!/bin/bash +PATH="$(yarn bin):$PATH" +export PATH +printf "Generate dependency graph for code" + +madge --image src.svg src +madge --image test.svg test \ No newline at end of file diff --git a/ssl/README.md b/ssl/README.md index 6c8f8e4f6..c4de3157f 100644 --- a/ssl/README.md +++ b/ssl/README.md @@ -39,6 +39,6 @@ to **servers/gateway/certs** directory. ### References -1. https://kb.mit.edu/confluence/display/istcontrib/Default+Certificate+Selection+in+Firefox -1. https://unix.stackexchange.com/questions/644176/how-to-permanently-add-self-signed-certificate-in-firefox -1. https://www.jscape.com/blog/firefox-client-certificate +1. [MIT knowledgebase article on default certificate selection in Firefox browser](https://kb.mit.edu/confluence/display/istcontrib/Default+Certificate+Selection+in+Firefox), +1. [Stackoverflow question on certificate installation in Firefox browser](https://unix.stackexchange.com/questions/644176/how-to-permanently-add-self-signed-certificate-in-firefox) +1. [Import client certificate into Firefox browser](https://www.jscape.com/blog/firefox-client-certificate)