Merge pull request #141 from royra/private-compose-tunnel-agent-service

make compose tunnel agent tunnel private
livecycle · Jul 24, 2023 · 2e2aaaa · 2e2aaaa
2 parents b9b8609 + 47d4f6e
commit 2e2aaaa
Show file tree

Hide file tree

Showing 15 changed files with 167 additions and 145 deletions.
diff --git a/.github/workflows/npm-publish-canary.yaml b/.github/workflows/npm-publish-canary.yaml
@@ -8,6 +8,7 @@ on:
 jobs:
   publish-packages:
     runs-on: ubuntu-latest
+    if: github.event.pull_request.head.repo.owner.name == 'livecycle'
 
     steps:
     - uses: actions/checkout@v3

diff --git a/packages/cli-common/src/lib/hooks.ts b/packages/cli-common/src/lib/hooks.ts
@@ -2,12 +2,16 @@ import { ComposeModel, FlatTunnel } from '@preevy/core'
 import { PluginContext } from './plugins/context'
 
 export type Hooks = {
+  filterUrls: {
+    args: FlatTunnel[]
+    return: FlatTunnel[]
+  }
   envCreated: {
     args: {
       envId: string
       urls: FlatTunnel[]
     }
-    return: { urls: FlatTunnel[] }
+    return: void
   }
   envDeleted: {
     args: {

diff --git a/packages/cli/src/commands/up.ts b/packages/cli/src/commands/up.ts
@@ -1,14 +1,13 @@
 import { Args, Flags, ux } from '@oclif/core'
 import {
-  commands, flattenTunnels, profileStore,
+  commands, profileStore,
   telemetryEmitter,
-  getUserCredentials, jwtGenerator, withBasicAuthCredentials,
 } from '@preevy/core'
-import { asyncReduce } from 'iter-tools-es'
 import { tunnelServerFlags } from '@preevy/cli-common'
 import { tunnelServerHello } from '../tunnel-server-client'
 import MachineCreationDriverCommand from '../machine-creation-driver-command'
 import { envIdFlags, urlFlags } from '../common-flags'
+import { filterUrls, printUrls } from './urls'
 
 // eslint-disable-next-line no-use-before-define
 export default class Up extends MachineCreationDriverCommand<typeof Up> {
@@ -62,7 +61,7 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
 
     const userModel = await this.ensureUserModel()
 
-    const { machine, tunnels, envId } = await commands.up({
+    const { machine, envId } = await commands.up({
       clientId,
       rootUrl,
       userSpecifiedServices: restArgs,
@@ -83,42 +82,34 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
       skipUnchangedFiles: flags['skip-unchanged-files'],
     })
 
-    let flatTunnels = flattenTunnels(tunnels)
-    if (flags['include-access-credentials']) {
-      const addCredentials = withBasicAuthCredentials(await getUserCredentials(jwtGenerator(tunnelingKey)))
-      flatTunnels = flatTunnels.map(t => Object.assign(t, { url: addCredentials(t.url) }))
-    }
+    this.log(`Preview environment ${envId} provisioned at: ${machine.locationDescription}`)
+
+    const flatTunnels = await commands.urls({
+      rootUrl,
+      clientId,
+      envId,
+      tunnelingKey,
+      includeAccessCredentials: flags['include-access-credentials'],
+    })
+
+    const urls = await filterUrls({
+      flatTunnels,
+      context: { log: this.logger, userModel },
+      filters: this.config.preevyHooks.filterUrls,
+    })
 
-    const result = await asyncReduce(
-      { urls: flatTunnels },
-      async ({ urls }, envCreated) => await envCreated(
+    await Promise.all(
+      this.config.preevyHooks.envCreated.map(envCreated => envCreated(
         { log: this.logger, userModel },
         { envId, urls },
-      ),
-      this.config.preevyHooks.envCreated,
+      )),
     )
 
     if (flags.json) {
-      return result.urls
+      return urls
     }
 
-    this.log(`Preview environment ${envId} provisioned at: ${machine.locationDescription}`)
-
-    // add credentials here
-
-    ux.table(
-      result.urls,
-      {
-        service: { header: 'Service' },
-        port: { header: 'Port' },
-        url: { header: 'URL' },
-      },
-      {
-        ...this.flags,
-        'no-truncate': this.flags['no-truncate'] ?? (!this.flags.output && !this.flags.csv && flags['include-access-credentials']),
-        'no-header': this.flags['no-header'] ?? (!this.flags.output && !this.flags.csv && flags['include-access-credentials']),
-      },
-    )
+    printUrls(urls, flags)
 
     return undefined
   }

diff --git a/packages/cli/src/commands/urls.ts b/packages/cli/src/commands/urls.ts
@@ -1,10 +1,40 @@
-import { Args, ux } from '@oclif/core'
-import { commands, findAmbientEnvId, getUserCredentials, jwtGenerator, profileStore, withBasicAuthCredentials } from '@preevy/core'
-import { tunnelServerFlags } from '@preevy/cli-common'
+import { Args, ux, Interfaces } from '@oclif/core'
+import { FlatTunnel, commands, findAmbientEnvId, profileStore } from '@preevy/core'
+import { HooksListeners, PluginContext, tunnelServerFlags } from '@preevy/cli-common'
+import { asyncReduce } from 'iter-tools-es'
 import { tunnelServerHello } from '../tunnel-server-client'
 import ProfileCommand from '../profile-command'
 import { envIdFlags, urlFlags } from '../common-flags'
 
+export const printUrls = (
+  flatTunnels: FlatTunnel[],
+  flags: Interfaces.InferredFlags<typeof ux.table.Flags> & { 'include-access-credentials': boolean },
+) => {
+  ux.table(
+    flatTunnels,
+    {
+      service: { header: 'Service' },
+      port: { header: 'Port' },
+      url: { header: 'URL' },
+    },
+    {
+      ...flags,
+      'no-truncate': flags['no-truncate'] ?? (!flags.output && !flags.csv && flags['include-access-credentials']),
+      'no-header': flags['no-header'] ?? (!flags.output && !flags.csv && flags['include-access-credentials']),
+    },
+  )
+}
+
+export const filterUrls = ({ flatTunnels, context, filters }: {
+  flatTunnels: FlatTunnel[]
+  context: PluginContext
+  filters: HooksListeners['filterUrls']
+}) => asyncReduce(
+  flatTunnels,
+  (urls, f) => f(context, urls),
+  filters,
+)
+
 // eslint-disable-next-line no-use-before-define
 export default class Urls extends ProfileCommand<typeof Urls> {
   static description = 'Show urls for an existing environment'
@@ -32,9 +62,7 @@ export default class Urls extends ProfileCommand<typeof Urls> {
   async run(): Promise<unknown> {
     const log = this.logger
     const { flags, args } = await this.parse(Urls)
-    const projectName = (await this.ensureUserModel()).name
-    log.debug(`project: ${projectName}`)
-    const envId = flags.id || await findAmbientEnvId(projectName)
+    const envId = flags.id || await findAmbientEnvId((await this.ensureUserModel()).name)
     log.debug(`envId: ${envId}`)
 
     const pStore = profileStore(this.store)
@@ -53,36 +81,29 @@ export default class Urls extends ProfileCommand<typeof Urls> {
       log: this.logger,
     })
 
-    let flatTunnels = await commands.urls({
+    const flatTunnels = await commands.urls({
       rootUrl,
       clientId,
       envId,
-      projectName,
       serviceAndPort: args.service ? { service: args.service, port: args.port } : undefined,
+      tunnelingKey,
+      includeAccessCredentials: flags['include-access-credentials'],
     })
 
-    if (flags['include-access-credentials']) {
-      const addCredentials = withBasicAuthCredentials(await getUserCredentials(jwtGenerator(tunnelingKey)))
-      flatTunnels = flatTunnels.map(t => Object.assign(t, { url: addCredentials(t.url) }))
-    }
+    const urls = await filterUrls({
+      flatTunnels,
+      context: {
+        log: this.logger,
+        userModel: { name: '' }, // TODO: don't want to require a compose file for this command
+      },
+      filters: this.config.preevyHooks.filterUrls,
+    })
 
     if (flags.json) {
-      return flatTunnels
+      return urls
     }
 
-    ux.table(
-      flatTunnels,
-      {
-        service: { header: 'Service' },
-        port: { header: 'Port' },
-        url: { header: 'URL' },
-      },
-      {
-        ...this.flags,
-        'no-truncate': this.flags['no-truncate'] ?? (!this.flags.output && !this.flags.csv && flags['include-access-credentials']),
-        'no-header': this.flags['no-header'] ?? (!this.flags.output && !this.flags.csv && flags['include-access-credentials']),
-      },
-    )
+    printUrls(urls, flags)
 
     return undefined
   }

diff --git a/packages/common/src/tunnel-name.ts b/packages/common/src/tunnel-name.ts
@@ -1,5 +1,4 @@
-const concat = (...v: (string | number)[]) => v.join('-')
-const tunnel = (port: number, v: (string | number)[]) => ({ port, tunnel: concat(...v).toLowerCase() })
+const tunnel = (port: number, v: (string | number)[]) => ({ port, tunnel: v.join('-').toLowerCase() })
 
 export type TunnelNameResolver = (x: {
   name: string

diff --git a/packages/compose-tunnel-agent/src/ssh/tunnel-client.ts b/packages/compose-tunnel-agent/src/ssh/tunnel-client.ts
@@ -15,7 +15,7 @@ type Forward = {
 export type Tunnel = {
   project: string
   service: string
-  ports: Record<number, string[]>
+  ports: Record<number, string>
 }
 
 export type SshState = {
@@ -121,7 +121,7 @@ export const sshClient = async ({
             service: service.name,
             project: service.project,
             ports: {},
-          }).ports[port] ||= []).push(url)
+          }).ports[port] = url)
           return res
         },
         {} as Record<string, Tunnel>,

diff --git a/packages/core/src/commands/up/index.ts b/packages/core/src/commands/up/index.ts
@@ -3,18 +3,16 @@ import fs from 'fs'
 import path from 'path'
 import { rimraf } from 'rimraf'
 import yaml from 'yaml'
-import { inspect } from 'util'
 import { TunnelOpts } from '../../ssh'
 import { ComposeModel, fixModelForRemote, getExposedTcpServicePorts, localComposeClient, resolveComposeFiles } from '../../compose'
 import { ensureCustomizedMachine } from './machine'
 import { wrapWithDockerSocket } from '../../docker'
 import { findAmbientEnvId } from '../../env-id'
-import { COMPOSE_TUNNEL_AGENT_SERVICE_NAME, addComposeTunnelAgentService, queryTunnels } from '../../compose-tunnel-agent-client'
-import { withSpinner } from '../../spinner'
+import { COMPOSE_TUNNEL_AGENT_SERVICE_NAME, addComposeTunnelAgentService } from '../../compose-tunnel-agent-client'
 import { MachineCreationDriver, MachineDriver, MachineBase } from '../../driver'
 import { remoteProjectDir } from '../../remote-files'
 import { Logger } from '../../log'
-import { Tunnel, tunnelUrlsForEnv } from '../../tunneling'
+import { tunnelUrlsForEnv } from '../../tunneling'
 import { FileToCopy, uploadWithSpinner } from '../../upload-files'
 
 const createCopiedFileInDataDir = (
@@ -99,7 +97,7 @@ const up = async ({
   allowedSshHostKeys: Buffer
   cwd: string
   skipUnchangedFiles: boolean
-}): Promise<{ machine: MachineBase; tunnels: Tunnel[]; envId: string }> => {
+}): Promise<{ machine: MachineBase; envId: string }> => {
   const projectName = userSpecifiedProjectName ?? userModel.name
   const remoteDir = remoteProjectDir(projectName)
 
@@ -109,7 +107,7 @@ const up = async ({
   // We start by getting the user model without injecting Preevy's environment
   // variables (e.g. `PREEVY_BASE_URI_BACKEND_3000`) so we can have the list of services
   // required to create said variables
-  const tunnelUrlsForService = tunnelUrlsForEnv({ projectName, envId, rootUrl: new URL(rootUrl), clientId })
+  const tunnelUrlsForService = tunnelUrlsForEnv({ envId, rootUrl: new URL(rootUrl), clientId })
   const composeEnv = { ...serviceLinkEnvVars(userModel, tunnelUrlsForService) }
 
   const composeFiles = await resolveComposeFiles({
@@ -143,28 +141,26 @@ const up = async ({
     machineDriver, machineCreationDriver, envId, log, debug,
   })
 
-  const { exec } = connection
-
-  const user = (
-    await exec('echo "$(id -u):$(stat -c %g /var/run/docker.sock)"')
-  ).stdout.trim()
+  try {
+    const { exec } = connection
 
-  const remoteModel = addComposeTunnelAgentService({
-    debug,
-    tunnelOpts,
-    urlSuffix: envId,
-    sshPrivateKeyPath: path.join(remoteDir, sshPrivateKeyFile.remote),
-    knownServerPublicKeyPath: path.join(remoteDir, knownServerPublicKey.remote),
-    user,
-  }, fixedModel)
+    const user = (
+      await exec('echo "$(id -u):$(stat -c %g /var/run/docker.sock)"')
+    ).stdout.trim()
 
-  const modelStr = yaml.stringify(remoteModel)
-  log.debug('model', modelStr)
-  const composeFilePath = (await createCopiedFile('docker-compose.yml', modelStr)).local
+    const remoteModel = addComposeTunnelAgentService({
+      debug,
+      tunnelOpts,
+      urlSuffix: envId,
+      sshPrivateKeyPath: path.join(remoteDir, sshPrivateKeyFile.remote),
+      knownServerPublicKeyPath: path.join(remoteDir, knownServerPublicKey.remote),
+      user,
+    }, fixedModel)
 
-  const withDockerSocket = wrapWithDockerSocket({ connection, log })
+    const modelStr = yaml.stringify(remoteModel)
+    log.debug('model', modelStr)
+    const composeFilePath = (await createCopiedFile('docker-compose.yml', modelStr)).local
 
-  try {
     await exec(`mkdir -p "${remoteDir}" && chown "${user}" "${remoteDir}"`, { asRoot: true })
 
     log.debug('Files to copy', filesToCopy)
@@ -173,26 +169,15 @@ const up = async ({
 
     const compose = localComposeClient({ composeFiles: [composeFilePath], projectName })
     const composeArgs = calcComposeArgs({ userSpecifiedServices, debug, cwd })
+
+    const withDockerSocket = wrapWithDockerSocket({ connection, log })
     log.debug('Running compose up with args: ', composeArgs)
     await withDockerSocket(() => compose.spawnPromise(composeArgs, { stdio: 'inherit' }))
-    const tunnels = await withSpinner(async () => {
-      const queryResult = await queryTunnels({
-        tunnelUrlsForService,
-        retryOpts: {
-          minTimeout: 1000,
-          maxTimeout: 2000,
-          retries: 10,
-          onFailedAttempt: e => { log.debug(`Failed to create tunnel: ${inspect(e)}`) },
-        },
-      })
-
-      return queryResult.tunnels
-    }, { opPrefix: 'Waiting for tunnels to be created' })
-
-    return { envId, machine, tunnels }
   } finally {
     await connection.close()
   }
+
+  return { envId, machine }
 }
 
 export default up
diff --git a/packages/core/src/commands/urls.ts b/packages/core/src/commands/urls.ts
@@ -1,17 +1,25 @@
+import { generateBasicAuthCredentials, jwtGenerator } from '../credentials'
 import { queryTunnels } from '../compose-tunnel-agent-client'
 import { flattenTunnels, tunnelUrlsForEnv } from '../tunneling'
 
-export const urls = async ({ envId,
-  rootUrl, clientId, projectName, serviceAndPort }: {
+export const urls = async ({ envId, rootUrl, clientId, serviceAndPort, tunnelingKey, includeAccessCredentials }: {
   envId: string
-  projectName: string
   rootUrl: string
   clientId: string
   serviceAndPort?: { service: string; port?: number }
+  tunnelingKey: string | Buffer
+  includeAccessCredentials: boolean
 }) => {
-  const tunnelUrlsForService = tunnelUrlsForEnv({ projectName, envId, rootUrl: new URL(rootUrl), clientId })
+  const tunnelUrlsForService = tunnelUrlsForEnv({ envId, rootUrl: new URL(rootUrl), clientId })
 
-  const { tunnels } = await queryTunnels({ tunnelUrlsForService, retryOpts: { retries: 2 } })
+  const credentials = await generateBasicAuthCredentials(jwtGenerator(tunnelingKey))
+
+  const { tunnels } = await queryTunnels({
+    tunnelUrlsForService,
+    retryOpts: { retries: 2 },
+    credentials,
+    includeAccessCredentials,
+  })
 
   return flattenTunnels(tunnels)
     .filter(tunnel => !serviceAndPort || (