Implements CLI options for runner and adds devContainer config (INTO-…

…CPS-Association#732)

  - Adds CLI interface to runner using commanderjs framework
  - Performs sanitary checks on the config file
  - Refactors code to reflect general purpose nature of runner
  - Adds better integration and E2E tests
  - Adds configuration for devContainers. This file can be used
    to create development environment. It is a good replacement
    for the script/ folder.

.codeclimate.yml

@@ -10,6 +10,7 @@ plugins:
       count_threshold: 3
       languages:
         typescript:
+          mass_threshold: 45
         javascript:
         python:
           python_version: 3

.devcontainer/devcontainer.json

@@ -0,0 +1,108 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node
+{
+	"name": "Digital Twin as a Service",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/typescript-node:1-20-bullseye",
+	"features": {
+		"ghcr.io/devcontainers-contrib/features/apt-get-packages:1": {
+			"clean_ppas": true,
+			"preserve_apt_list": true,
+			"packages": "curl graphviz htop net-tools powerline"
+		},
+		"ghcr.io/devcontainers/features/common-utils:2": {
+			"installZsh": true,
+			"configureZshAsDefaultShell": true,
+			"installOhMyZsh": true,
+			"installOhMyZshConfig": true,
+			"username": "devcontainer",
+			"userUid": "1001",
+			"userGid": "automatic"
+		},
+		"ghcr.io/devcontainers-contrib/features/exa:1": {
+			"version": "latest"
+		},
+		"ghcr.io/devcontainers/features/git:1": {
+			"version": "os-provided"
+		},
+		"ghcr.io/devcontainers-contrib/features/markdownlint-cli:1": {
+			"version": "latest"
+		},
+		"ghcr.io/devcontainers-contrib/features/mkdocs:2": {
+			"version": "latest",
+			"plugins": "mkdocs-material pymdown-extensions mkdocstrings[crystal,python] mkdocs-monorepo-plugin mkdocs-pdf-export-plugin mkdocs-awesome-pages-plugin python-markdown-math mkdocs-open-in-new-tab mkdocs-with-pdf qrcode"
+		},
+		"ghcr.io/devcontainers-contrib/features/nestjs-cli:2": {
+			"version": "latest"
+		},
+		"ghcr.io/devcontainers-contrib/features/npm-package:1": {
+			"package": "madge"
+		},
+		"ghcr.io/devcontainers-contrib/features/pipx-package:1": {},
+		"ghcr.io/devcontainers-contrib/features/poetry:2": {
+			"version": "latest"
+		},
+		"ghcr.io/devcontainers-contrib/features/pre-commit:2": {
+			"version": "latest"
+		},
+		"ghcr.io/devcontainers-contrib/features/tmux-apt-get:1": {},
+		"ghcr.io/devcontainers-contrib/features/typescript:2": {
+			"version": "latest"
+		},
+		"ghcr.io/devcontainers-contrib/features/vercel-serve:1": {
+			"version": "latest"
+		},
+		"ghcr.io/devcontainers-contrib/features/zsh-plugins:0": {
+			"plugins": "ssh-agent npm zsh-autosuggestions",
+			"omzPlugins": "https://github.com/zsh-users/zsh-autosuggestions",
+			"username": "node"
+		}
+	},
+  "customizations": {
+    "vscode": {
+      "extensions": [
+				"AlexShen.classdiagram-ts",
+				"christian-kohler.npm-intellisense",
+				"DavidAnson.vscode-markdownlin",
+				"donjayamanne.githistory",
+				"eamodio.gitlens",
+				"humao.rest-client",
+				"johnpapa.vscode-peacock",
+				"mhutchie.git-graph",
+				"SimonSiefke.svg-preview",
+				"redhat.vscode-yaml",
+				"VisualStudioExptTeam.vscodeintellicode",
+				"vscode-icons-team.vscode-icons"
+			],
+      "settings": {
+        "editor.tabSize": 2,
+        "terminal.integrated.defaultProfile.linux": "zsh",
+        "terminal.integrated.profiles.linux": {
+          "bash": {
+            "path": "bash",
+            "icon": "terminal-bash"
+          },
+          "zsh": {
+            "path": "zsh"
+          },
+          "tmux": {
+            "path": "tmux",
+            "icon": "terminal-tmux"
+          },
+          "pwsh": {
+            "path": "pwsh",
+            "icon": "terminal-powershell"
+          }
+        }
+      }
+    }
+  },
+	"forwardPorts": [
+    4000,		//react client
+    4001,		// lib microservice
+    5000,		// DT runner
+		8000		// mkdocs
+	]
+}
+// Execute after login:
+//	source /usr/share/powerline/bindings/zsh/powerline.zsh

servers/execution/runner/DEVELOPER.md

@@ -18,6 +18,12 @@ yarn start      # Start the application
 yarn clean      # Deletes directories "build", "coverage", and "dist"
 ```
 
+### On Filenames in Tests
+
+The jest and nestjs combination can not detect tests in files
+with _config_ in their names. Hence, the config word has been
+replaced with _options_ in the names of test files.
+
 ## :package: :ship: NPM package
 
 ### Github Package Registry

servers/execution/runner/README.md

@@ -35,7 +35,7 @@ The template configuration file is:
 
 ```ini
 port: 5000
-location: "lifecycle" #directory location of scripts
+location: "script" #directory location of scripts
 ```
 
 The file should be named as _runner.yaml_ and placed in the directory

servers/execution/runner/jest.config.json

@@ -16,16 +16,15 @@
     "node_modules",
     "./dist",
     "../src/app.module.ts",
-    "../src/main.ts",
-    "../src/runner.ts"
+    "../src/main.ts"
   ],
   "extensionsToTreatAsEsm": [".ts"],
   "moduleFileExtensions": [
     "js",
     "json",
     "ts"
   ],
-  "modulePathIgnorePatterns": ["config"],
+  "modulePathIgnorePatterns": [],
   "coverageDirectory": "<rootDir>/coverage/",
   "coverageThreshold": {
     "global": {

servers/execution/runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@into-cps-association/runner",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "DT Runner",
   "main": "dist/src/runner.js",
   "repository": "https://github.com/into-cps-association/DTaaS.git",
@@ -11,20 +11,20 @@
   "scripts": {
     "build": "npx tsc",
     "clean": "npx rimraf build node_modules coverage dist src.svg test.svg",
+    "check:final": "concurrently -s all -g -n syntax,format,graph,build,test \"yarn syntax\" \"yarn format\" \"yarn graph\" \"yarn build\" \"yarn test\"",
     "format": "prettier --ignore-path ../.gitignore --write \"**/*.{ts,tsx,css,scss}\"",
     "start": "npx cross-env NODE_OPTIONS='--es-module-specifier-resolution=node --experimental-specifier-resolution=node' NODE_NO_WARNINGS=1 node dist/src/main.js",
     "syntax": "npx eslint . --fix",
-    "pretest": "npx rimraf runner.yaml scripts && npx shx cp test/config/runner.yaml runner.yaml && npx shx mkdir scripts && npx shx cp test/data/scripts/* scripts/.",
-    "posttest": "npx rimraf runner.yaml scripts",
+    "pretest": "npx rimraf runner.test.yaml script && npx shx cp test/config/runner.test.yaml runner.test.yaml && npx shx mkdir script && npx shx cp test/data/script/* script/.",
+    "posttest": "npx rimraf runner.test.yaml script",
     "test": "npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=true",
-    "test:e2e": "yarn pretest && npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=true --verbose=true test/app.e2e.spec.ts && yarn posttest",
-    "test:nocov": "npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=false",
-    "test:watchAll": "npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=true --watchAll",
+    "test:e2e": "yarn pretest && npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=true --verbose=true test/e2e && yarn posttest",
+    "test:int": "yarn pretest && npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=true --verbose=true test/integration && yarn posttest",
+    "test:nocov": "yarn pretest && npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=false && yarn posttest",
+    "test:unit": "yarn pretest && npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=true --verbose=true test/unit && yarn posttest",
+    "test:watchAll": "yarn pretest && npx cross-env NODE_OPTIONS=--experimental-vm-modules NODE_NO_WARNINGS=1 jest --coverage=true --watchAll && yarn posttest",
     "graph": "npx madge --image src.svg src && npx madge --image test.svg test"
   },
-  "script-unused": {
-    "runner": "npx cross-env NODE_OPTIONS='--es-module-specifier-resolution=node --experimental-modules --experimental-specifier-resolution=node' NODE_NO_WARNINGS=1 node dist/src/runner.js"
-  },
   "bin": {
     "runner": "./dist/src/main.js"
   },
@@ -48,6 +48,7 @@
     "@types/supertest": "^6.0.2",
     "@typescript-eslint/eslint-plugin": "^7.6.0",
     "@typescript-eslint/parser": "^7.6.0",
+    "concurrently": "^8.2.2",
     "eslint": "^8.57.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-config-prettier": "^9.1.0",
@@ -69,12 +70,16 @@
     "@nestjs/config": "^3.2.2",
     "@nestjs/core": "^10.3.7",
     "@nestjs/platform-express": "^10.3.7",
+    "chalk": "^5.3.0",
+    "commander": "^12.0.0",
     "cross-env": "^7.0.3",
     "execa": "^8.0.1",
     "express": "^4.19.2",
     "js-yaml": "^4.1.0",
+    "keyv": "^4.5.4",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",
     "zod": "^3.22.4"
-  }
+  },
+  "packageManager": "[email protected]+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610"
 }

servers/execution/runner/runner.yaml

@@ -1,2 +1,3 @@
 port: 5000
-location: "scripts" #directory location of scripts
+location: 'script' #directory location of scripts
+commands: []

servers/execution/runner/runner.yaml.sample

@@ -1,2 +1,6 @@
 port: 5000
-location: "scripts" #directory location of scripts
+location: 'script' #directory location of scripts
+commands:
+  - create
+  - execute
+  - terminate

servers/execution/runner/src/app.controller.ts

@@ -9,7 +9,7 @@ import {
 } from '@nestjs/common';
 import { Response } from 'express';
 import { CommandStatus } from './interfaces/command.interface.js';
-import ExecaManager from './execaManager.service.js';
+import ExecaManager from './execa-manager.service.js';
 import { ExecuteCommandDto, executeCommandSchema } from './dto/command.dto.js';
 import ZodValidationPipe from './validation.pipe.js';
 
@@ -19,13 +19,13 @@ export default class AppController {
   constructor(private readonly manager: ExecaManager) {} // eslint-disable-line no-empty-function
 
   @Get('history')
-  getHello(): Array<ExecuteCommandDto> {
+  getHistory(): Array<ExecuteCommandDto> {
     return this.manager.checkHistory();
   }
 
   @Post()
   @UsePipes(new ZodValidationPipe(executeCommandSchema))
-  async changePhase(
+  async newCommand(
     @Body() executeCommandDto: ExecuteCommandDto,
     @Res({ passthrough: true }) res: Response,
   ): Promise<void> {

servers/execution/runner/src/app.module.ts

@@ -1,18 +1,13 @@
 import { Module } from '@nestjs/common';
-import { ConfigModule } from '@nestjs/config';
+import RunnerFactory from './runner-factory.service.js';
 import AppController from './app.controller.js';
-import ExecaManager from './execaManager.service.js';
+import ExecaManager from './execa-manager.service.js';
 import Queue from './queue.service.js';
-import configuration from './config/configuration.js';
+import Config from './config/configuration.service.js';
 
 @Module({
-  imports: [
-    ConfigModule.forRoot({
-      ignoreEnvFile: true,
-      load: [configuration],
-    }),
-  ],
+  imports: [],
   controllers: [AppController],
-  providers: [ExecaManager, Queue],
+  providers: [ExecaManager, Queue, RunnerFactory, Config],
 })
 export default class AppModule {}

servers/execution/runner/src/config/commander.ts

@@ -0,0 +1,42 @@
+import { Command } from 'commander';
+import { readFileSync } from 'fs';
+import chalk from 'chalk';
+import Keyv from 'keyv';
+import resolveFile from './util.js';
+
+export function createCommand(name: string): [Command, Keyv] {
+  return [new Command(name), new Keyv()];
+}
+
+export default async function CLI(
+  program: Command,
+  CLIOptions: Keyv,
+): Promise<Keyv> {
+  program
+    .description('Remote code execution for humans')
+    .option(
+      '-c --config <string>',
+      'runner config file specified in yaml format',
+      'runner.yaml',
+    )
+    .helpOption('-h --help', 'display help')
+    .showHelpAfterError()
+    .helpInformation();
+
+  program.parse(process.argv);
+  const options = program.opts();
+
+  if (options.config !== undefined) {
+    const configFile: string = options.config;
+    const resolvedFilename: string = resolveFile(configFile.toString());
+    try {
+      readFileSync(resolvedFilename, 'utf8');
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.log(chalk.bold.redBright('Config file can not be read. Exiting'));
+      throw new Error('Invalid configuration');
+    }
+    await CLIOptions.set('configFile', resolvedFilename);
+  }
+  return CLIOptions;
+}

servers/execution/runner/src/config/config.interface.ts

@@ -0,0 +1,20 @@
+// unused type
+export type PermitCommand = {
+  name: string;
+  executable: string;
+};
+
+export type ConfigValues = {
+  port: number;
+  // os-compatible relative filepath of the config yaml file.
+  // The relative filepath is with reference to the execution
+  // directory of the runner
+  location: string;
+  commands: Array<string>;
+};
+
+export const configDefault: ConfigValues = {
+  port: 5000,
+  location: 'script',
+  commands: [],
+};

servers/execution/runner/src/config/configuration.service.ts

@@ -0,0 +1,35 @@
+import { readFileSync } from 'fs';
+import path from 'node:path';
+import * as yaml from 'js-yaml';
+import { Injectable } from '@nestjs/common';
+import Keyv from 'keyv';
+import { configDefault, ConfigValues } from './config.interface.js';
+import resolveFile from './util.js';
+
+@Injectable()
+export default class Config {
+  private configValues: ConfigValues = configDefault;
+
+  async loadConfig(CLIOptions: Keyv): Promise<void> {
+    const configFile = await CLIOptions.get('configFile');
+    if (configFile !== undefined) {
+      this.configValues = yaml.load(
+        readFileSync(resolveFile(configFile.toString()), 'utf8'),
+      ) as ConfigValues;
+    }
+    const dir = path.dirname(resolveFile(configFile.toString()));
+    this.configValues.location = path.join(dir, this.configValues.location);
+  }
+
+  permitCommands(): Array<string> {
+    return this.configValues.commands;
+  }
+
+  getPort(): number {
+    return this.configValues.port;
+  }
+
+  getLocation(): string {
+    return this.configValues.location;
+  }
+}