diff --git a/backend/package.json b/backend/package.json index b702e97..050e3bc 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,6 +11,7 @@ "start:localmultiuservm": "ts-node src/LocalMultiuserVMApplication.ts", "start:openstack": "ts-node src/OpenStackApplication.ts", "start:docker": "ts-node src/DockerApplication.ts", + "start:firecracker": "ts-node --files src/FirecrackerApplication.ts", "lint": "eslint src/**/**/*.{js,ts,tsx}", "compile": "tsc --noEmit" }, @@ -25,8 +26,9 @@ "@types/dockerode": "^3.3.15", "@types/express": "^4.17.17", "@types/jest": "^29.4.0", - "@types/jsonwebtoken": "^9.0.1", + "@types/jsonwebtoken": "^9.0.2", "@types/mongodb": "^3.6.20", + "@types/netmask": "^2.0.2", "@types/node": "^18.14.5", "@types/ssh2": "^0.5.52", "@types/supertest": "^2.0.10", @@ -41,7 +43,7 @@ "supertest": "^6.3.3", "ts-jest": "^29.0.5", "ts-node": "^10.9.1", - "typescript": "^4.9.5" + "typescript": "^5.1.6" }, "dependencies": { "axios": "^0.26.1", @@ -52,10 +54,11 @@ "dockerode": "^3.3.4", "jsonwebtoken": "^9.0.0", "mongodb": "^3.7.3", + "netmask": "^2.0.2", "node-pty": "^0.10.1", "npm": "^9.6.2", "path-to-regexp": "^6.2.1", - "ssh2": "^1.11.0", + "ssh2": "^1.14.0", "toad-scheduler": "^2.2.0", "ws": "^8.13.0" } diff --git a/backend/src/FirecrackerApplication.ts b/backend/src/FirecrackerApplication.ts new file mode 100644 index 0000000..86c7db4 --- /dev/null +++ b/backend/src/FirecrackerApplication.ts @@ -0,0 +1,15 @@ +import api from "./Api"; +import serverCreator from "./Server"; +import MongoDBPersister from "./database/MongoDBPersister"; +import MongoDBAuthenticationProvider from "./authentication/MongoDBAuthenticationProvider"; +import FirecrackerProvider from "./providers/FirecrackerProvider"; + +const persister = new MongoDBPersister(process.env.MONGODB_URL); +console.log("Attempting to start Firecracker Application."); +serverCreator( + api( + persister, + [new MongoDBAuthenticationProvider(persister)], + new FirecrackerProvider() + ) +); diff --git a/backend/src/providers/DockerProvider.ts b/backend/src/providers/DockerProvider.ts index c83c256..116f780 100644 --- a/backend/src/providers/DockerProvider.ts +++ b/backend/src/providers/DockerProvider.ts @@ -376,7 +376,28 @@ export default class DockerProvider implements InstanceProvider { timestampCreated + " and should be deleted" ); - Environment.deleteInstanceEnvironments(container.Id); + this.deleteServer(container.Id) + .then(() => { + console.log( + "Deleted expired container" + + container.Names + + " expiration date: " + + timestampCreated + + " deadline: " + + deadline + ); + Environment.deleteInstanceEnvironments(container.Id); + }) + .catch((err) => { + return reject( + new Error( + "DockerProvider: Failed to delete container to be pruned. " + + container.Names + + " " + + err + ) + ); + }); } } }); @@ -408,7 +429,7 @@ export default class DockerProvider implements InstanceProvider { rejected = true; return reject( new Error( - "DockerProvider: Container was not stated. Check image and cmd used to init the container." + "DockerProvider: Container was not started. Check image and cmd used to init the container." ) ); } else { @@ -472,10 +493,12 @@ export default class DockerProvider implements InstanceProvider { const sshConn = new Client(); sshConn .on("ready", () => { + sshConn.end(); resolved = true; return resolve(); }) .on("error", (err) => { + sshConn.end(); console.log( "DockerProvider: SSH connection failed - retrying..." + err ); @@ -485,6 +508,7 @@ export default class DockerProvider implements InstanceProvider { port: port, username: process.env.SSH_USERNAME, password: process.env.SSH_PASSWORD, + readyTimeout: 1000, }); await providerInstance.sleep(1000); timeout -= 1; diff --git a/backend/src/providers/FirecrackerProvider.ts b/backend/src/providers/FirecrackerProvider.ts new file mode 100644 index 0000000..1d933c3 --- /dev/null +++ b/backend/src/providers/FirecrackerProvider.ts @@ -0,0 +1,791 @@ +import { + InstanceProvider, + VMEndpoint, + InstanceNotFoundErrorMessage, +} from "./Provider"; +import axios, { AxiosInstance } from "axios"; +import { ToadScheduler, SimpleIntervalJob, AsyncTask } from "toad-scheduler"; +import { Client } from "ssh2"; +import Environment from "../Environment"; +import fs from "fs/promises"; +import { Netmask } from "netmask"; +import { ChildProcess, exec, spawn } from "child_process"; + +const defaultAxiosTimeout = 30000; +const schedulerIntervalSeconds = 5 * 60; + +type microVMId = string; + +interface FirecrackerLogger { + log_path: string; + level?: string; + show_level?: boolean; + show_log_origin?: boolean; +} + +interface FirecrackerMachineConfig { + vcpu_count: number; + mem_size_mib: number; + ht_enabled?: boolean; + track_dirty_pages?: boolean; + smt?: boolean; + cpu_template?: "T2" | "T2S" | "T2CL" | "T2A" | "V1N1" | "None"; +} + +interface FirecrackerBalloonMemory { + amount_mib: number; + deflate_on_oom: boolean; + stats_polling_interval_s: number; +} + +interface FirecrackerBootSource { + kernel_image_path: string; + boot_args: string; +} + +interface FirecrackerDrive { + drive_id: string; + path_on_host: string; + is_root_device: boolean; + is_read_only: boolean; +} + +interface FirecrackerNetworkInterface { + iface_id: string; + host_dev_name: string; + guest_mac?: string; + //rx_rate_limiter?: RateLimiter + //tx_rate_limiter?: RateLimiter; +} + +interface FirecrackerActionState { + state: string; +} + +class microVMInstance { + vmEndpoint: VMEndpoint; + expirationDate: Date; + tapInterfaceId: string; + username: string; + groupNumber: number; + environment: string; + process: ChildProcess; + microVM: microVM; +} + +class microVM { + binPath: string; + socketPath: string; + axiosInstance: AxiosInstance; + baseURL: string; + microVMInstance: microVMInstance; + + constructor(binPath: string, socketPath: string) { + this.binPath = binPath; + this.socketPath = socketPath; + + this.axiosInstance = axios.create({ + timeout: defaultAxiosTimeout, + socketPath: socketPath, + }); + this.axiosInstance.defaults.headers.common["Accept"] = "application/json"; + this.axiosInstance.defaults.headers.common["Content-Type"] = + "application/json"; + this.baseURL = "http://localhost"; + + this.microVMInstance = new microVMInstance(); + } + + spawn(): Promise { + return new Promise((resolve, reject) => { + try { + this.microVMInstance.process = spawn( + this.binPath, + ["--api-sock", this.socketPath], + { + detached: true, + } + ); + } catch (err) { + return reject( + new Error("FirecrackerProvider: Could not spawn firecracker process") + ); + } + + this.microVMInstance.process.on("exit", async function () { + await fs.unlink(this.socketPath).catch(async (err) => { + // if ENOENT, file does not exist as intended + if ("ENOENT" !== err.code) { + return reject( + new Error( + "FirecrackerProvider: Could not cleanup socketPath " + err + ) + ); + } + }); + }); + + this.microVMInstance.process.on("close", async function () { + await fs.unlink(this.socketPath).catch(async (err) => { + // if ENOENT, file does not exist as intended + if ("ENOENT" !== err.code) { + return reject( + new Error( + "FirecrackerProvider: Could not cleanup socketPath " + err + ) + ); + } + }); + }); + + this.microVMInstance.process.on("error", async function () { + await fs.unlink(this.socketPath).catch(async (err) => { + if ("ENOENT" !== err.code) { + return reject( + new Error( + "FirecrackerProvider: Could not cleanup socketPath " + err + ) + ); + } + // if ENOENT, file does not exist as intended + }); + }); + + resolve(this.microVMInstance.process); + }); + } + + kill(): boolean { + return this.microVMInstance.process.kill(); + } + + async setLogger(data: FirecrackerLogger): Promise { + return new Promise((resolve, reject) => { + this.axiosInstance + .put(this.baseURL + "/logger", data) + .then((response) => { + resolve(response.data); + }) + .catch((err) => { + reject(err); + }); + }); + } + + async setMachineConfig( + data: FirecrackerMachineConfig + ): Promise { + return new Promise((resolve, reject) => { + this.axiosInstance + .put(this.baseURL + "/machine-config", data) + .then((response) => { + resolve(response.data); + }) + .catch((err) => { + reject(err); + }); + }); + } + + async setBalloonMemory( + data: FirecrackerBalloonMemory + ): Promise { + return new Promise((resolve, reject) => { + this.axiosInstance + .put(this.baseURL + "/balloon", data) + .then((response) => { + resolve(response.data); + }) + .catch((err) => { + reject(err); + }); + }); + } + + async setBootSource( + data: FirecrackerBootSource + ): Promise { + return new Promise((resolve, reject) => { + this.axiosInstance + .put(this.baseURL + "/boot-source", data) + .then((response) => { + resolve(response.data); + }) + .catch((err) => { + reject(err); + }); + }); + } + + async addDrive( + driveId: string, + data: FirecrackerDrive + ): Promise { + return new Promise((resolve, reject) => { + this.axiosInstance + .put(this.baseURL + "/drives/" + driveId, data) + .then((response) => { + resolve(response.data); + }) + .catch((err) => { + reject(err); + }); + }); + } + + async addNetworkInterface( + ifId: string, + data: FirecrackerNetworkInterface + ): Promise { + return new Promise((resolve, reject) => { + this.axiosInstance + .put(this.baseURL + "/network-interfaces/" + ifId, data) + .then((response) => { + resolve(response.data); + }) + .catch((err) => { + reject(err); + }); + }); + } + + async invokeAction(actionType: string): Promise { + return new Promise((resolve, reject) => { + this.axiosInstance + .put(this.baseURL + "/actions", { action_type: actionType }) + .then((response) => { + resolve(response.data); + }) + .catch((err) => { + reject(err); + }); + }); + } +} + +export default class FirecrackerProvider implements InstanceProvider { + private firecrackers: Map; + + // Firecracker config + private binPath: string; + private socketPathPrefix: string; + private kernelImage: string; + private kernelBootARGs: string; + private rootFSDrive: string; + private vcpuCount: number; + private memSizeMiB: number; + private memBalloonSizeMiB: number; + + // currently IPv4 only ;) what a shame ;) + private networkCIDR: Netmask; + private availableIpAddresses: Array; + private bridgeInterface: string; + + // Firecracker Provider config + private maxInstanceLifetimeMinutes: number; + private microVMSSHTimeoutSeconds = 60; + + // SSH and LanguageServer Port config + private sshPort: number; + private lsPort: number; + + private providerInstance: FirecrackerProvider; + + constructor() { + this.binPath = process.env.FIRECRACKER_BIN_PATH ?? "/usr/bin/firecracker"; + this.socketPathPrefix = process.env.FIRECRACKER_SOCKET_PATH_PREFIX; + this.kernelImage = process.env.FIRECRACKER_KERNEL_IMAGE; + this.kernelBootARGs = process.env.FIRECRACKER_KERNEL_BOOT_ARGS; + this.rootFSDrive = process.env.FIRECRACKER_ROOTFS_DRIVE; + this.networkCIDR = new Netmask(process.env.FIRECRACKER_NETWORK_CIDR); + this.bridgeInterface = process.env.FIRECRACKER_BRIDGE_INTERFACE; + this.vcpuCount = parseInt(process.env.FIRECRACKER_VCPU_COUNT ?? "2"); + this.memSizeMiB = parseInt(process.env.FIRECRACKER_MEM_SIZE_MIB ?? "2048"); + this.memBalloonSizeMiB = parseInt( + process.env.FIRECRACKER_MEM_BALLOON_SIZE_MIB ?? + new Number(this.memSizeMiB - 512).toString() + ); + + const bridgeInterfaceRegExp = new RegExp(/^[a-z0-9]+$/i); + if (bridgeInterfaceRegExp.test(this.bridgeInterface) === false) { + throw new Error( + "Invalid FIRECRACKER_BRIDGE_INTERFACE. Needs to be an alphanumeric string." + ); + } + this.availableIpAddresses = new Array(); + this.networkCIDR.forEach((ip) => { + // reserve the first and last IP address for the host + if (ip !== this.networkCIDR.first && ip !== this.networkCIDR.last) + this.availableIpAddresses.push(ip); + }); + + this.maxInstanceLifetimeMinutes = parseInt( + process.env.FIRECRACKER_MAX_INSTANCE_LIFETIME_MINUTES + ); + + this.providerInstance = this; + + this.firecrackers = new Map(); + + // better use env var to allow configuration of port numbers? + this.sshPort = 22; + this.lsPort = 3005; + + const scheduler = new ToadScheduler(); + + const task = new AsyncTask( + "FirecrackerProvider Instance Pruning Task", + () => { + return this.pruneMicroVMInstance().then(() => { + //console.log("FirecrackerProvider: Pruning finished..."); + }); + }, + (err: Error) => { + console.log( + "FirecrackerProvider: Could not prune stale microVM instances..." + + err + ); + } + ); + const job = new SimpleIntervalJob( + { seconds: schedulerIntervalSeconds, runImmediately: true }, + task + ); + + scheduler.addSimpleIntervalJob(job); + } + + async createServer( + username: string, + groupNumber: number, + environment: string, + kernelImage?: string, + rootFSDrive?: string + ): Promise { + const providerInstance = this.providerInstance; + const socketPath = + this.socketPathPrefix + "_" + username + "-" + environment; + + return new Promise(async (resolve, reject) => { + await fs.unlink(socketPath).catch(async (err) => { + if ("ENOENT" !== err.code) { + return reject( + new Error( + "FirecrackerProvider: Could not cleanup socketPath " + err + ) + ); + } + // if ENOENT, file does not exist as intended + }); + + const microVMKernelImage = kernelImage ?? providerInstance.kernelImage; + const microVMKernelBootARGs = + kernelImage ?? providerInstance.kernelBootARGs; + const microVMRootFSDrive = rootFSDrive ?? providerInstance.rootFSDrive; + + const logFileName = + "/tmp/firecracker.log" + "_" + username + "-" + environment; + const logFileTime = new Date(); + + await fs.unlink(socketPath).catch(async (err) => { + if ("ENOENT" !== err.code) { + return reject( + new Error("FirecrackerProvider: Could not cleanup logFile " + err) + ); + } + // if ENOENT, file does not exist as intended + }); + await fs + .utimes(logFileName, logFileTime, logFileTime) + .catch(async (err) => { + if ("ENOENT" !== err.code) { + return reject( + new Error("FirecrackerProvider: Could not touch logfile " + err) + ); + } + // if ENOENT, file does not exist as intended, open and close it to update access time + const fh = await fs.open(logFileName, "a"); + await fh.close(); + }); + + // jailer is recommended for production + const mv = new microVM(this.binPath, socketPath); + await mv.spawn().then(async (process) => { + console.log("Started new firecracker process " + process.pid); + + await mv + .setLogger({ + log_path: + "/tmp/firecracker.log" + "_" + username + "-" + environment, + level: "debug", + show_level: true, + show_log_origin: true, + }) + .then(async () => { + await mv + .setMachineConfig({ + vcpu_count: this.vcpuCount, + mem_size_mib: this.memSizeMiB, + }) + .then(async () => { + await mv + .setBalloonMemory({ + amount_mib: this.memBalloonSizeMiB, + deflate_on_oom: true, + stats_polling_interval_s: 1, + }) + .then(async () => { + await mv + .setBootSource({ + kernel_image_path: microVMKernelImage, + boot_args: microVMKernelBootARGs, + }) + .then(async () => { + await mv + .addDrive("rootfs", { + drive_id: "rootfs", + path_on_host: microVMRootFSDrive, + is_root_device: true, + is_read_only: false, + }) + .then(async () => { + // create tap dev on host + const microVMIPID = + this.availableIpAddresses.length; + const microVMIPAddress = + this.availableIpAddresses.pop(); + const hexStringIP = microVMIPAddress + .split(".") + .map((value) => + Number(value).toString(16).padStart(2, "0") + ) + .join(":"); + const microVMMACAddress = + "f6:" + + this.networkCIDR.bitmask.toString(16) + + ":" + + hexStringIP; + const iface_id = "net1"; + const tap_id = "fctap" + microVMIPID; + console.log( + "FirecrackerProvider: creating tap device: " + + tap_id + + " with IP: " + + microVMIPAddress + + " and MAC: " + + microVMMACAddress + ); + + // create tap interface and attach it to the bridge + exec( + "sudo ip tuntap add " + + tap_id + + " mode tap && sudo ip link set " + + tap_id + + " up && sudo brctl addif " + + this.bridgeInterface + + " " + + tap_id, + (error, stdout, stderr) => { + if (error) { + mv.kill(); + return reject( + new Error( + "FirecrackerProvider: Unable to create TAP device." + + stderr + + " " + + stdout + ) + ); + } + } + ); + // wait for tap dev to be setup + await providerInstance.sleep(1000); + + // need to setup tap devices etc. on the host in advance + // fcnet-setup.sh or similar can be used in rootfs + // expects mac with prefix f6:, rest of the address is used as + // mask and ipv4 address in hex + await mv + .addNetworkInterface(iface_id, { + iface_id: iface_id, + guest_mac: microVMMACAddress, + host_dev_name: tap_id, + }) + .then(async () => { + // possibly wait for all previous changes (async) + // see getting_started doc for firecracker + await mv + .invokeAction("InstanceStart") + .then(() => { + console.log( + "FirecrackerProvider: microVM started" + ); + // wait for ssh + const expirationDate = new Date( + Date.now() + + providerInstance.maxInstanceLifetimeMinutes * + 60 * + 1000 + ); + providerInstance + .waitForServerSSH( + microVMIPAddress, + providerInstance.sshPort, + providerInstance.microVMSSHTimeoutSeconds + ) + .then(() => { + console.log( + "FirecrackerProvider: microVM SSH ready" + ); + const vmEndpoint = { + instance: microVMIPAddress, + providerInstanceStatus: + "Environment will be deleted at " + + expirationDate.toLocaleString(), + IPAddress: microVMIPAddress, + SSHPort: providerInstance.sshPort, + LanguageServerPort: + providerInstance.lsPort, + }; + this.firecrackers.set( + microVMIPAddress, + { + vmEndpoint: vmEndpoint, + microVM: mv, + process: process, + expirationDate: expirationDate, + tapInterfaceId: tap_id, + username: username, + groupNumber: groupNumber, + environment: environment, + } + ); + + return resolve(vmEndpoint); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not connect to microVM using SSH " + + err + ) + ); + }); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not invoke Action to start microVM: " + + err.response.data.fault_message + ) + ); + }); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not add NetworkInterface: " + + err.response.data.fault_message + ) + ); + }); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not add Drive: " + + err.response.data.fault_message.fault_message + ) + ); + }); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not set BootSource: " + + err.response.data.fault_message + ) + ); + }); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not set BalloonMemory: " + + err.response.data.fault_message + ) + ); + }); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not set MachineConfig: " + + err.response.data.fault_message + ) + ); + }); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not create Logger: " + + err.response.data.fault_message + ) + ); + }); + }); + }); + } + + async getServer(instance: string): Promise { + return new Promise((resolve, reject) => { + const vmEndpoint = this.firecrackers.get(instance)?.vmEndpoint; + if (vmEndpoint !== undefined) { + return resolve(vmEndpoint); + } else { + return reject(new Error(InstanceNotFoundErrorMessage)); + } + }); + } + + async deleteServer(instance: string): Promise { + return new Promise(async (resolve, reject) => { + const providerInstance = this.providerInstance; + const fc = this.firecrackers.get(instance); + const fi = fc?.process; + const vmEndpoint = this.firecrackers.get(instance)?.vmEndpoint; + if (vmEndpoint !== undefined && fc !== undefined && fi !== undefined) { + // wait for stop tasks to end + await providerInstance.sleep(1000); + // https://github.com/firecracker-microvm/firecracker/blob/main/docs/api_requests/actions.md#send-ctrlaltdel + await fc.microVM.invokeAction("SendCtrlAltDel").then(async () => { + console.log("FirecrackerProvider: microVM stopped"); + // wait for stop tasks to end, is this still necessary after SendCtrlAltDel? + if (fi.kill()) { + // wait for process to be killed properly before deleting the tap dev + await providerInstance.sleep(1000); + + // delete tap interface + exec( + "sudo brctl delif " + + this.bridgeInterface + + " " + + fc.tapInterfaceId + + " && sudo ip tuntap del " + + fc.tapInterfaceId + + " mode tap", + (error, stdout, stderr) => { + if (error) { + return reject( + new Error( + "FirecrackerProvider: Unable to remove TAP device." + + stderr + + " " + + stdout + ) + ); + } + } + ); + this.availableIpAddresses.push(vmEndpoint.IPAddress); + this.firecrackers.delete(instance); + + return resolve(); + } else { + return reject( + new Error( + "FirecrackerProvider: Could not delete instance: " + instance + ) + ); + } + }); + } else { + return reject(new Error(InstanceNotFoundErrorMessage)); + } + }); + } + + async pruneMicroVMInstance(): Promise { + console.log("FirecrackerProvider: Pruning stale microVM instances..."); + + return new Promise((resolve, reject) => { + // get microVMs older than timestamp + const deadline = new Date(Date.now()); + console.log( + "FirecrackerProvider: Pruning microVM instances older than " + + deadline.toISOString() + ); + this.firecrackers?.forEach((microVM, microVMId) => { + if (microVM.expirationDate < deadline) { + this.deleteServer(microVMId) + .then(() => { + console.log( + "FirecrackerProvider: deleted expired microVM: " + + microVMId + + " expiration date: " + + microVM.expirationDate + ); + Environment.deleteInstanceEnvironments(microVMId); + }) + .catch((err) => { + return reject( + new Error( + "FirecrackerProvider: Could not delete expired instance during pruning: " + + microVMId + + " " + + err + ) + ); + }); + } + }); + return resolve(); + }); + } + + waitForServerSSH(ip: string, port: number, timeout: number): Promise { + const providerInstance = this.providerInstance; + + return new Promise(async (resolve, reject) => { + let resolved = false; + // check ssh connection + while (timeout > 0 && resolved === false) { + const sshConn = new Client(); + sshConn + .on("ready", () => { + resolved = true; + sshConn.end(); + return resolve(); + }) + .on("error", (err) => { + sshConn.end(); + console.log( + "FirecrackerProvider: SSH connection failed - retrying... " + err + ); + }) + .connect({ + host: ip, + port: port, + username: process.env.SSH_USERNAME, + password: process.env.SSH_PASSWORD, + readyTimeout: 1000, + }); + await providerInstance.sleep(1000); + timeout -= 1; + } + if (!resolved) + return reject( + "FirecrackerProvider: Timed out waiting for SSH connection." + ); + }); + } + + sleep(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); + } +} diff --git a/backend/src/providers/OpenStackProvider.ts b/backend/src/providers/OpenStackProvider.ts index bb89a35..96cb69a 100644 --- a/backend/src/providers/OpenStackProvider.ts +++ b/backend/src/providers/OpenStackProvider.ts @@ -367,7 +367,11 @@ export default class OpenStackProvider implements InstanceProvider { .then(function () { // instance creation takes longer after new image is used, needs to be copied to compute hosts providerInstance - .waitForServerSSH(floatingIpAddress, 300) + .waitForServerSSH( + floatingIpAddress, + providerInstance.sshPort, + 300 + ) .then(() => { // finished, successfully created server and associated floating ip return resolve({ @@ -419,7 +423,11 @@ export default class OpenStackProvider implements InstanceProvider { } else { // instance creation takes longer after new image is used, needs to be copied to compute hosts providerInstance - .waitForServerSSH(fixedIpAddress, 300) + .waitForServerSSH( + fixedIpAddress, + providerInstance.sshPort, + 300 + ) .then(() => { // finished, successfully created server and got fixed ip return resolve({ @@ -719,35 +727,40 @@ export default class OpenStackProvider implements InstanceProvider { }); } - waitForServerSSH(address: string, timeout: number): Promise { + waitForServerSSH(ip: string, port: number, timeout: number): Promise { const providerInstance = this.providerInstance; return new Promise(async (resolve, reject) => { - let resolved: boolean; + let resolved = false; // check ssh connection - while (timeout > 0 && resolved === undefined) { + while (timeout > 0 && resolved === false) { const sshConn = new Client(); sshConn .on("ready", () => { - return resolve(); resolved = true; + sshConn.end(); + return resolve(); }) .on("error", (err) => { + sshConn.end(); console.log( - "OpenStackProvider: SSH connection failed - retrying..." + err + "OpenStackProvider: SSH connection failed - retrying... " + err ); }) .connect({ - host: address, - // better use env var to allow configuration of port numbers? - port: 22, + host: ip, + port: port, username: process.env.SSH_USERNAME, password: process.env.SSH_PASSWORD, + readyTimeout: 1000, }); - await providerInstance.sleep(5000); - timeout -= 5; + await providerInstance.sleep(1000); + timeout -= 1; } - return reject("OpenStackProvider: Timed out waiting for SSH connection."); + if (!resolved) + return reject( + "OpenStackProvider: Timed out waiting for SSH connection." + ); }); } diff --git a/package-lock.json b/package-lock.json index 28d8ca3..9f7e8c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,10 +26,11 @@ "dockerode": "^3.3.4", "jsonwebtoken": "^9.0.0", "mongodb": "^3.7.3", + "netmask": "^2.0.2", "node-pty": "^0.10.1", "npm": "^9.6.2", "path-to-regexp": "^6.2.1", - "ssh2": "^1.11.0", + "ssh2": "^1.14.0", "toad-scheduler": "^2.2.0", "ws": "^8.13.0" }, @@ -42,8 +43,9 @@ "@types/dockerode": "^3.3.15", "@types/express": "^4.17.17", "@types/jest": "^29.4.0", - "@types/jsonwebtoken": "^9.0.1", + "@types/jsonwebtoken": "^9.0.2", "@types/mongodb": "^3.6.20", + "@types/netmask": "^2.0.2", "@types/node": "^18.14.5", "@types/ssh2": "^0.5.52", "@types/supertest": "^2.0.10", @@ -58,7 +60,7 @@ "supertest": "^6.3.3", "ts-jest": "^29.0.5", "ts-node": "^10.9.1", - "typescript": "^4.9.5" + "typescript": "^5.1.6" } }, "backend/node_modules/@jest/console": { @@ -1132,6 +1134,19 @@ } } }, + "backend/node_modules/typescript": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", + "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "backend/node_modules/yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", @@ -5530,9 +5545,9 @@ "license": "MIT" }, "node_modules/@types/jsonwebtoken": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.1.tgz", - "integrity": "sha512-c5ltxazpWabia/4UzhIoaDcIza4KViOQhdbjRlfcIGVnsE3c3brkz9Z+F/EeJIECOQP7W7US2hNE930cWWkPiw==", + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", + "integrity": "sha512-drE6uz7QBKq1fYqqoFKTDRdFCPHd5TCub75BM+D+cMx7NU9hUz7SESLfC2fSCXVFMO5Yj8sOWHuGqPgjc+fz0Q==", "dev": true, "dependencies": { "@types/node": "*" @@ -5572,6 +5587,12 @@ "version": "0.7.31", "license": "MIT" }, + "node_modules/@types/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-Hsg8tvhb7IWX64zjlrKmg0PZeNoP0FzYqjHD4jvTNjHU3YfLlK4GG9xW1K0hrFR5YkMGbh18fc2csykdCgpfzQ==", + "dev": true + }, "node_modules/@types/node": { "version": "17.0.35", "license": "MIT" @@ -7411,7 +7432,9 @@ "license": "MIT" }, "node_modules/buildcheck": { - "version": "0.0.3", + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", "optional": true, "engines": { "node": ">=10.0.0" @@ -8035,12 +8058,14 @@ } }, "node_modules/cpu-features": { - "version": "0.0.4", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.8.tgz", + "integrity": "sha512-BbHBvtYhUhksqTjr6bhNOjGgMnhwhGTQmOoZGD+K7BCaQDCuZl/Ve1ZxUSMRwVC4D/rkCPQ2MAIeYzrWyK7eEg==", "hasInstallScript": true, "optional": true, "dependencies": { - "buildcheck": "0.0.3", - "nan": "^2.15.0" + "buildcheck": "~0.0.6", + "nan": "^2.17.0" }, "engines": { "node": ">=10.0.0" @@ -15112,8 +15137,9 @@ } }, "node_modules/nan": { - "version": "2.16.0", - "license": "MIT" + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", + "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==" }, "node_modules/nanoid": { "version": "3.3.4", @@ -15149,6 +15175,14 @@ "version": "2.6.2", "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/no-case": { "version": "3.0.4", "dev": true, @@ -22161,20 +22195,20 @@ "license": "BSD-3-Clause" }, "node_modules/ssh2": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.11.0.tgz", - "integrity": "sha512-nfg0wZWGSsfUe/IBJkXVll3PEZ//YH2guww+mP88gTpuSU4FtZN7zu9JoeTGOyCNx2dTDtT9fOpWwlzyj4uOOw==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.14.0.tgz", + "integrity": "sha512-AqzD1UCqit8tbOKoj6ztDDi1ffJZ2rV2SwlgrVVrHPkV5vWqGJOVp5pmtj18PunkPJAuKQsnInyKV+/Nb2bUnA==", "hasInstallScript": true, "dependencies": { - "asn1": "^0.2.4", + "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2" }, "engines": { "node": ">=10.16.0" }, "optionalDependencies": { - "cpu-features": "~0.0.4", - "nan": "^2.16.0" + "cpu-features": "~0.0.8", + "nan": "^2.17.0" } }, "node_modules/stable": { @@ -26991,9 +27025,9 @@ "dev": true }, "@types/jsonwebtoken": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.1.tgz", - "integrity": "sha512-c5ltxazpWabia/4UzhIoaDcIza4KViOQhdbjRlfcIGVnsE3c3brkz9Z+F/EeJIECOQP7W7US2hNE930cWWkPiw==", + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", + "integrity": "sha512-drE6uz7QBKq1fYqqoFKTDRdFCPHd5TCub75BM+D+cMx7NU9hUz7SESLfC2fSCXVFMO5Yj8sOWHuGqPgjc+fz0Q==", "dev": true, "requires": { "@types/node": "*" @@ -27028,6 +27062,12 @@ "@types/ms": { "version": "0.7.31" }, + "@types/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-Hsg8tvhb7IWX64zjlrKmg0PZeNoP0FzYqjHD4jvTNjHU3YfLlK4GG9xW1K0hrFR5YkMGbh18fc2csykdCgpfzQ==", + "dev": true + }, "@types/node": { "version": "17.0.35" }, @@ -28250,7 +28290,9 @@ "version": "1.1.2" }, "buildcheck": { - "version": "0.0.3", + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", "optional": true }, "builtin-modules": { @@ -28659,11 +28701,13 @@ } }, "cpu-features": { - "version": "0.0.4", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.8.tgz", + "integrity": "sha512-BbHBvtYhUhksqTjr6bhNOjGgMnhwhGTQmOoZGD+K7BCaQDCuZl/Ve1ZxUSMRwVC4D/rkCPQ2MAIeYzrWyK7eEg==", "optional": true, "requires": { - "buildcheck": "0.0.3", - "nan": "^2.15.0" + "buildcheck": "~0.0.6", + "nan": "^2.17.0" } }, "create-require": { @@ -33465,7 +33509,9 @@ } }, "nan": { - "version": "2.16.0" + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", + "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==" }, "nanoid": { "version": "3.3.4", @@ -33488,6 +33534,11 @@ "neo-async": { "version": "2.6.2" }, + "netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==" + }, "no-case": { "version": "3.0.4", "dev": true, @@ -35595,8 +35646,9 @@ "@types/dockerode": "^3.3.15", "@types/express": "^4.17.17", "@types/jest": "^29.4.0", - "@types/jsonwebtoken": "^9.0.1", + "@types/jsonwebtoken": "^9.0.2", "@types/mongodb": "^3.6.20", + "@types/netmask": "^2.0.2", "@types/node": "^18.14.5", "@types/ssh2": "^0.5.52", "@types/supertest": "^2.0.10", @@ -35615,17 +35667,18 @@ "jest": "^29.4.3", "jsonwebtoken": "^9.0.0", "mongodb": "^3.7.3", + "netmask": "^2.0.2", "node-pty": "^0.10.1", "npm": "^9.6.2", "path-to-regexp": "^6.2.1", "prettier": "^2.8.4", - "ssh2": "^1.11.0", + "ssh2": "^1.14.0", "supertest": "^6.3.3", "toad-scheduler": "^2.2.0", "ts-jest": "^29.0.5", "ts-node": "^10.9.1", - "typescript": "^4.9.5", - "ws": "^8.13.0" + "typescript": "^5.1.6", + "ws": "^8.12.1" }, "dependencies": { "@jest/console": { @@ -36443,6 +36496,12 @@ "yargs-parser": "^21.0.1" } }, + "typescript": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", + "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==", + "dev": true + }, "yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", @@ -38803,14 +38862,14 @@ "dev": true }, "ssh2": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.11.0.tgz", - "integrity": "sha512-nfg0wZWGSsfUe/IBJkXVll3PEZ//YH2guww+mP88gTpuSU4FtZN7zu9JoeTGOyCNx2dTDtT9fOpWwlzyj4uOOw==", + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.14.0.tgz", + "integrity": "sha512-AqzD1UCqit8tbOKoj6ztDDi1ffJZ2rV2SwlgrVVrHPkV5vWqGJOVp5pmtj18PunkPJAuKQsnInyKV+/Nb2bUnA==", "requires": { - "asn1": "^0.2.4", + "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2", - "cpu-features": "~0.0.4", - "nan": "^2.16.0" + "cpu-features": "~0.0.8", + "nan": "^2.17.0" } }, "stable": {