fix: use pre-built binary instead of go run in e2e tests (coder#16236)

Using `go run` inside of a test is fragile, because it means we have to
wait for `go` to compile the binary while also constrained on resources
by the fact that Playwright and coderd are already running. We should
instead compile a coder binary for the current platform before the tests
and use it directly.

Author: aslilac

.github/workflows/ci.yaml

@@ -704,6 +704,9 @@ jobs:
       - run: make gen/mark-fresh
         name: make gen
 
+      - run: make site/e2e/bin/coder
+        name: make coder
+
       - run: pnpm build
         env:
           NODE_OPTIONS: ${{ github.repository_owner == 'coder' && '--max_old_space_size=8192' || '' }}

.gitignore

@@ -36,6 +36,7 @@ site/.swc
 .gen-golden
 
 # Build
+bin/
 build/
 dist/
 out/

Makefile

@@ -944,7 +944,12 @@ test-clean:
 	go clean -testcache
 .PHONY: test-clean
 
-test-e2e: site/node_modules/.installed site/out/index.html
+site/e2e/bin/coder: go.mod go.sum $(GO_SRC_FILES)
+	go build -o $@ \
+		-tags ts_omit_aws,ts_omit_bird,ts_omit_tap,ts_omit_kube \
+		./enterprise/cmd/coder
+
+test-e2e: site/e2e/bin/coder site/node_modules/.installed site/out/index.html
 	cd site/
 ifdef CI
 	DEBUG=pw:api pnpm playwright:test --forbid-only --workers 1

site/e2e/constants.ts

@@ -1,6 +1,6 @@
 import * as path from "node:path";
 
-export const coderMain = path.join(__dirname, "../../enterprise/cmd/coder");
+export const coderBinary = path.join(__dirname, "./bin/coder");
 
 // Default port from the server
 export const coderPort = process.env.CODER_E2E_PORT

site/e2e/helpers.ts

@@ -15,7 +15,7 @@ import * as ssh from "ssh2";
 import { TarWriter } from "utils/tar";
 import {
 	agentPProfPort,
-	coderMain,
+	coderBinary,
 	coderPort,
 	defaultOrganizationName,
 	defaultPassword,
@@ -311,12 +311,9 @@ export const createGroup = async (page: Page): Promise<string> => {
 export const sshIntoWorkspace = async (
 	page: Page,
 	workspace: string,
-	binaryPath = "go",
+	binaryPath = coderBinary,
 	binaryArgs: string[] = [],
 ): Promise<ssh.Client> => {
-	if (binaryPath === "go") {
-		binaryArgs = ["run", coderMain];
-	}
 	const sessionToken = await findSessionToken(page);
 	return new Promise<ssh.Client>((resolve, reject) => {
 		const cp = spawn(binaryPath, [...binaryArgs, "ssh", "--stdio", workspace], {
@@ -398,7 +395,7 @@ export const startAgent = async (
 	page: Page,
 	token: string,
 ): Promise<ChildProcess> => {
-	return startAgentWithCommand(page, token, "go", "run", coderMain);
+	return startAgentWithCommand(page, token, coderBinary);
 };
 
 /**
@@ -479,27 +476,21 @@ export const startAgentWithCommand = async (
 		},
 	});
 	cp.stdout.on("data", (data: Buffer) => {
-		console.info(
-			`[agent] [stdout] [onData] ${data.toString().replace(/\n$/g, "")}`,
-		);
+		console.info(`[agent][stdout] ${data.toString().replace(/\n$/g, "")}`);
 	});
 	cp.stderr.on("data", (data: Buffer) => {
-		console.info(
-			`[agent] [stderr] [onData] ${data.toString().replace(/\n$/g, "")}`,
-		);
+		console.info(`[agent][stderr] ${data.toString().replace(/\n$/g, "")}`);
 	});
 
 	await page
 		.getByTestId("agent-status-ready")
-		.waitFor({ state: "visible", timeout: 45_000 });
+		.waitFor({ state: "visible", timeout: 15_000 });
 	return cp;
 };
 
-export const stopAgent = async (cp: ChildProcess, goRun = true) => {
-	// When the web server is started with `go run`, it spawns a child process with coder server.
-	// `pkill -P` terminates child processes belonging the same group as `go run`.
-	// The command `kill` is used to terminate a web server started as a standalone binary.
-	exec(goRun ? `pkill -P ${cp.pid}` : `kill ${cp.pid}`, (error) => {
+export const stopAgent = async (cp: ChildProcess) => {
+	// The command `kill` is used to terminate an agent started as a standalone binary.
+	exec(`kill ${cp.pid}`, (error) => {
 		if (error) {
 			throw new Error(`exec error: ${JSON.stringify(error)}`);
 		}
@@ -922,10 +913,8 @@ export const updateTemplate = async (
 
 	const sessionToken = await findSessionToken(page);
 	const child = spawn(
-		"go",
+		coderBinary,
 		[
-			"run",
-			coderMain,
 			"templates",
 			"push",
 			"--test.provisioner",

site/e2e/playwright.config.ts

@@ -1,8 +1,6 @@
-import { execSync } from "node:child_process";
 import * as path from "node:path";
 import { defineConfig } from "@playwright/test";
 import {
-	coderMain,
 	coderPort,
 	coderdPProfPort,
 	e2eFakeExperiment1,
@@ -13,45 +11,12 @@ import {
 
 export const wsEndpoint = process.env.CODER_E2E_WS_ENDPOINT;
 
-// If running terraform tests, verify the requirements exist in the
-// environment.
-//
-// These execs will throw an error if the status code is non-zero.
-// So if both these work, then we can launch terraform provisioners.
-let hasTerraform = false;
-let hasDocker = false;
-try {
-	execSync("terraform --version");
-	hasTerraform = true;
-} catch {
-	/* empty */
-}
-
-try {
-	execSync("docker --version");
-	hasDocker = true;
-} catch {
-	/* empty */
-}
-
-if (!hasTerraform || !hasDocker) {
-	const msg = `Terraform provisioners require docker & terraform binaries to function. \n${
-		hasTerraform
-			? ""
-			: "\tThe `terraform` executable is not present in the runtime environment.\n"
-	}${
-		hasDocker
-			? ""
-			: "\tThe `docker` executable is not present in the runtime environment.\n"
-	}`;
-	throw new Error(msg);
-}
-
 const localURL = (port: number, path: string): string => {
 	return `http://localhost:${port}${path}`;
 };
 
 export default defineConfig({
+	globalSetup: require.resolve("./setup/preflight"),
 	projects: [
 		{
 			name: "testsSetup",
@@ -84,7 +49,8 @@ export default defineConfig({
 	webServer: {
 		url: `http://localhost:${coderPort}/api/v2/deployment/config`,
 		command: [
-			`go run -tags embed ${coderMain} server`,
+			`go run -tags embed ${path.join(__dirname, "../../enterprise/cmd/coder")}`,
+			"server",
 			"--global-config $(mktemp -d -t e2e-XXXXXXXXXX)",
 			`--access-url=http://localhost:${coderPort}`,
 			`--http-address=0.0.0.0:${coderPort}`,

site/e2e/proxy.ts

@@ -1,11 +1,11 @@
 import { type ChildProcess, exec, spawn } from "node:child_process";
-import { coderMain, coderPort, workspaceProxyPort } from "./constants";
+import { coderBinary, coderPort, workspaceProxyPort } from "./constants";
 import { waitUntilUrlIsNotResponding } from "./helpers";
 
 export const startWorkspaceProxy = async (
 	token: string,
 ): Promise<ChildProcess> => {
-	const cp = spawn("go", ["run", coderMain, "wsproxy", "server"], {
+	const cp = spawn(coderBinary, ["wsproxy", "server"], {
 		env: {
 			...process.env,
 			CODER_PRIMARY_ACCESS_URL: `http://127.0.0.1:${coderPort}`,
@@ -26,8 +26,8 @@ export const startWorkspaceProxy = async (
 	return cp;
 };
 
-export const stopWorkspaceProxy = async (cp: ChildProcess, goRun = true) => {
-	exec(goRun ? `pkill -P ${cp.pid}` : `kill ${cp.pid}`, (error) => {
+export const stopWorkspaceProxy = async (cp: ChildProcess) => {
+	exec(`kill ${cp.pid}`, (error) => {
 		if (error) {
 			throw new Error(`exec error: ${JSON.stringify(error)}`);
 		}

site/e2e/setup/createUsers.spec.ts renamed to site/e2e/setup/addUsersAndLicense.spec.ts

@@ -0,0 +1,45 @@
+import { execSync } from "node:child_process";
+import * as path from "node:path";
+
+export default function () {
+	// If running terraform tests, verify the requirements exist in the
+	// environment.
+	//
+	// These execs will throw an error if the status code is non-zero.
+	// So if both these work, then we can launch terraform provisioners.
+	let hasTerraform = false;
+	let hasDocker = false;
+	try {
+		execSync("terraform --version");
+		hasTerraform = true;
+	} catch {
+		/* empty */
+	}
+
+	try {
+		execSync("docker --version");
+		hasDocker = true;
+	} catch {
+		/* empty */
+	}
+
+	if (!hasTerraform || !hasDocker) {
+		const msg = `Terraform provisioners require docker & terraform binaries to function. \n${
+			hasTerraform
+				? ""
+				: "\tThe `terraform` executable is not present in the runtime environment.\n"
+		}${
+			hasDocker
+				? ""
+				: "\tThe `docker` executable is not present in the runtime environment.\n"
+		}`;
+		throw new Error(msg);
+	}
+
+	if (!process.env.CI) {
+		console.info("==> make site/e2e/bin/coder");
+		execSync("make site/e2e/bin/coder", {
+			cwd: path.join(__dirname, "../../../"),
+		});
+	}
+}

site/e2e/setup/preflight.ts

@@ -18,8 +18,6 @@ test.beforeEach(async ({ page }) => {
 });
 
 test("app", async ({ context, page }) => {
-	test.setTimeout(75_000);
-
 	const appContent = "Hello World";
 	const token = randomUUID();
 	const srv = http
@@ -64,7 +62,7 @@ test("app", async ({ context, page }) => {
 
 	// Wait for the web terminal to open in a new tab
 	const pagePromise = context.waitForEvent("page");
-	await page.getByText(appName).click({ timeout: 60_000 });
+	await page.getByText(appName).click({ timeout: 10_000 });
 	const app = await pagePromise;
 	await app.waitForLoadState("domcontentloaded");
 	await app.getByText(appContent).isVisible();

site/e2e/tests/app.spec.ts

@@ -64,5 +64,5 @@ test(`ssh with agent ${agentVersion}`, async ({ page }) => {
 	});
 
 	await stopWorkspace(page, workspaceName);
-	await stopAgent(agent, false);
+	await stopAgent(agent);
 });

site/e2e/tests/outdatedAgent.spec.ts

@@ -21,8 +21,6 @@ test.beforeEach(async ({ page }) => {
 });
 
 test(`ssh with client ${clientVersion}`, async ({ page }) => {
-	test.setTimeout(60_000);
-
 	const token = randomUUID();
 	const template = await createTemplate(page, {
 		apply: [

site/e2e/tests/outdatedCLI.spec.ts

@@ -16,8 +16,6 @@ test.beforeEach(async ({ page }) => {
 });
 
 test("web terminal", async ({ context, page }) => {
-	test.setTimeout(75_000);
-
 	const token = randomUUID();
 	const template = await createTemplate(page, {
 		apply: [