cta: docker API (#149)

* compose tunnel agent: add docker API
  - passthrough API to docker on port 3001
  - websocket impl for exec and logs
* add option to show preevy_proxy url
* tunnel-server: fix error handling
* tunnel-server: fix ws auth

packages/cli-common/src/lib/flags.ts

@@ -52,4 +52,8 @@ export const urlFlags = {
     description: 'Include access credentials for basic auth for each service URL',
     default: false,
   }),
+  'show-preevy-service-urls': Flags.boolean({
+    description: 'Show URLs for internal Preevy services',
+    default: false,
+  }),
 } as const

packages/cli/src/commands/up.ts

@@ -91,6 +91,7 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
       envId,
       tunnelingKey,
       includeAccessCredentials: flags['include-access-credentials'],
+      showPreevyService: flags['show-preevy-service-urls'],
       retryOpts: {
         minTimeout: 1000,
         maxTimeout: 2000,

packages/cli/src/commands/urls.ts

@@ -88,6 +88,7 @@ export default class Urls extends ProfileCommand<typeof Urls> {
       serviceAndPort: args.service ? { service: args.service, port: args.port } : undefined,
       tunnelingKey,
       includeAccessCredentials: flags['include-access-credentials'],
+      showPreevyService: flags['show-preevy-service-urls'],
       retryOpts: { retries: 2 },
     })
 

packages/compose-tunnel-agent/Dockerfile.dev

@@ -0,0 +1,3 @@
+FROM node:18-alpine as development
+WORKDIR /app
+CMD [ "yarn", "-s", "dev" ]

packages/compose-tunnel-agent/docker-compose.override.yml

@@ -4,8 +4,7 @@ services:
   preevy_proxy:
     build:
       context: .
-      target: development
+      dockerfile: Dockerfile.dev
     volumes:
       - ${HOME}/.ssh:/root/.ssh
 
-

packages/compose-tunnel-agent/docker-compose.yml

@@ -17,6 +17,7 @@ services:
 
     ports:
       - 3000
+      - 3001
 
     # healthcheck:
     #   test: wget --no-verbose --tries=1 --spider http://localhost:3000/healthz || exit 1

packages/compose-tunnel-agent/index.ts

@@ -2,16 +2,21 @@ import fs from 'fs'
 import path from 'path'
 import Docker from 'dockerode'
 import { inspect } from 'node:util'
+import http from 'node:http'
 import { rimraf } from 'rimraf'
 import pino from 'pino'
 import pinoPretty from 'pino-pretty'
 import { EOL } from 'os'
 import { ConnectionCheckResult, requiredEnv, checkConnection, formatPublicKey, parseSshUrl, SshConnectionConfig, tunnelNameResolver } from '@preevy/common'
 import createDockerClient from './src/docker'
-import createWebServer from './src/web'
+import createApiServerHandler from './src/http/api-server'
 import { sshClient as createSshClient } from './src/ssh'
+import { createDockerProxyHandlers } from './src/http/docker-proxy'
+import { tryHandler, tryUpgradeHandler } from './src/http/http-server-helpers'
+import { httpServerHandlers } from './src/http'
 
 const homeDir = process.env.HOME || '/root'
+const dockerSocket = '/var/run/docker.sock'
 
 const readDir = async (dir: string) => {
   try {
@@ -70,11 +75,11 @@ const formatConnectionCheckResult = (
 
 const writeLineToStdout = (s: string) => [s, EOL].forEach(d => process.stdout.write(d))
 
-const main = async () => {
-  const log = pino({
-    level: process.env.DEBUG || process.env.DOCKER_PROXY_DEBUG ? 'debug' : 'info',
-  }, pinoPretty({ destination: pino.destination(process.stderr) }))
+const log = pino({
+  level: process.env.DEBUG || process.env.DOCKER_PROXY_DEBUG ? 'debug' : 'info',
+}, pinoPretty({ destination: pino.destination(process.stderr) }))
 
+const main = async () => {
   const { connectionConfig, sshUrl } = await sshConnectionConfigFromEnv()
 
   log.debug('ssh config: %j', {
@@ -92,20 +97,21 @@ const main = async () => {
     process.exit(0)
   }
 
-  const docker = new Docker({ socketPath: '/var/run/docker.sock' })
+  const docker = new Docker({ socketPath: dockerSocket })
   const dockerClient = createDockerClient({ log: log.child({ name: 'docker' }), docker, debounceWait: 500 })
 
+  const sshLog = log.child({ name: 'ssh' })
   const sshClient = await createSshClient({
     connectionConfig,
     tunnelNameResolver: tunnelNameResolver({ userDefinedSuffix: process.env.TUNNEL_URL_SUFFIX }),
-    log: log.child({ name: 'ssh' }),
+    log: sshLog,
     onError: err => {
       log.error(err)
       process.exit(1)
     },
   })
 
-  log.info('ssh client connected to %j', sshUrl)
+  sshLog.info('ssh client connected to %j', sshUrl)
   let currentTunnels = dockerClient.getRunningServices().then(services => sshClient.updateTunnels(services))
 
   void dockerClient.startListening({
@@ -115,25 +121,50 @@ const main = async () => {
     },
   })
 
-  const listenAddress = process.env.PORT ?? 3000
-  if (typeof listenAddress === 'string' && Number.isNaN(Number(listenAddress))) {
-    await rimraf(listenAddress)
+  const apiListenAddress = process.env.PORT ?? 3000
+  if (typeof apiListenAddress === 'string' && Number.isNaN(Number(apiListenAddress))) {
+    await rimraf(apiListenAddress)
   }
 
-  const webServer = createWebServer({
-    log: log.child({ name: 'web' }),
-    currentSshState: async () => (
-      await currentTunnels
-    ),
+  const { handler, upgradeHandler } = httpServerHandlers({
+    log: log.child({ name: 'http' }),
+    apiHandler: createApiServerHandler({
+      log: log.child({ name: 'api' }),
+      currentSshState: async () => (await currentTunnels),
+    }),
+    dockerProxyHandlers: createDockerProxyHandlers({
+      log: log.child({ name: 'docker-proxy' }),
+      dockerSocket,
+      docker,
+    }),
+    dockerProxyPrefix: '/docker/',
   })
-    .listen(listenAddress, () => {
-      log.info(`listening on ${inspect(webServer.address())}`)
+
+  const httpLog = log.child({ name: 'http' })
+
+  const httpServer = http.createServer(tryHandler({ log: httpLog }, async (req, res) => {
+    httpLog.debug('request %s %s', req.method, req.url)
+    return await handler(req, res)
+  }))
+    .on('upgrade', tryUpgradeHandler({ log: httpLog }, async (req, socket, head) => {
+      httpLog.debug('upgrade %s %s', req.method, req.url)
+      return await upgradeHandler(req, socket, head)
+    }))
+    .listen(apiListenAddress, () => {
+      httpLog.info(`API server listening on ${inspect(httpServer.address())}`)
     })
     .on('error', err => {
-      log.error(err)
+      httpLog.error(err)
       process.exit(1)
     })
     .unref()
 }
 
-void main()
+void main();
+
+['SIGTERM', 'SIGINT'].forEach(signal => {
+  process.once(signal, async () => {
+    log.info(`shutting down on ${signal}`)
+    process.exit(0)
+  })
+})

packages/compose-tunnel-agent/jest.config.js

@@ -0,0 +1,5 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

packages/compose-tunnel-agent/package.json

@@ -17,31 +17,38 @@
     "pino": "^8.11.0",
     "pino-pretty": "^9.4.0",
     "rimraf": "^5.0.0",
-    "ssh2": "^1.12.0"
+    "ssh2": "^1.12.0",
+    "ws": "^8.13.0"
   },
   "devDependencies": {
+    "@jest/globals": "^29.5.0",
     "@types/dockerode": "^3.3.14",
+    "@types/http-proxy": "^1.17.9",
     "@types/lodash": "^4.14.192",
     "@types/node": "18",
+    "@types/node-fetch": "^2.6.3",
     "@types/shell-escape": "^0.2.1",
     "@types/ssh2": "^1.11.8",
     "@typescript-eslint/eslint-plugin": "^5.55.0",
     "@typescript-eslint/parser": "^5.55.0",
     "esbuild": "^0.17.14",
     "eslint": "^8.36.0",
     "husky": "^8.0.0",
+    "jest": "^29.4.3",
     "lint-staged": "^13.1.2",
+    "node-fetch": "2.6.9",
     "tsx": "^3.12.3",
-    "typescript": "^5.0.4"
+    "typescript": "^5.0.4",
+    "wait-for-expect": "^3.0.2"
   },
   "scripts": {
     "start": "node out/index.js",
     "dev": "tsx watch ./index.ts",
     "lint": "eslint . --ext .ts,.tsx --cache",
     "clean": "rm -rf dist out",
-    "build": "node --version && node build.mjs",
+    "build": "yarn tsc --noEmit && node build.mjs",
     "prepack": "yarn build",
     "prepare": "cd ../.. && husky install",
-    "bump-to": "yarn version --no-commit-hooks --no-git-tag-version --new-version"
+    "test": "yarn jest"
   }
 }

packages/compose-tunnel-agent/src/docker.ts

@@ -60,6 +60,7 @@ const client = ({
       stream.on('data', handler)
       log.info('listening on docker')
       void handler()
+      return { close: () => stream.removeAllListeners() }
     },
   }
 }

packages/compose-tunnel-agent/src/http/api-server.ts

@@ -0,0 +1,25 @@
+import url from 'node:url'
+import { Logger } from '@preevy/common'
+import { SshState } from '../ssh'
+import { NotFoundError, respondAccordingToAccept, respondJson, tryHandler } from './http-server-helpers'
+
+const createApiServerHandler = ({ log, currentSshState }: {
+  log: Logger
+  currentSshState: ()=> Promise<SshState>
+}) => tryHandler({ log }, async (req, res) => {
+  const { pathname: path } = url.parse(req.url || '')
+
+  if (path === '/tunnels') {
+    respondJson(res, await currentSshState())
+    return
+  }
+
+  if (path === '/healthz') {
+    respondAccordingToAccept(req, res, 'OK')
+    return
+  }
+
+  throw new NotFoundError()
+})
+
+export default createApiServerHandler