chore: codspeed benchmark

.github/workflows/continuous_benchmark.yml

@@ -0,0 +1,50 @@
+name: Continuous benchmark
+
+on:
+  workflow_dispatch:
+  pull_request: 
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+env:
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
+  FORCE_COLOR: true
+  CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }}
+  CODSPEED: true
+
+jobs:
+  codspeed:
+    if: ${{ github.repository_owner == 'withastro' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Setup PNPM
+        uses: pnpm/action-setup@v3
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        run: pnpm run build
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@b587655f756aab640e742fec141261bc6f0a569d # v3.0.1
+        timeout-minutes: 30
+        with:
+          run: pnpm benchmark codspeed
+

benchmark/bench/_util.js

@@ -14,7 +14,7 @@ export const astroBin = path.resolve(astroPkgPath, '../astro.js');
 export function calculateStat(numbers) {
 	const avg = numbers.reduce((a, b) => a + b, 0) / numbers.length;
 	const stdev = Math.sqrt(
-		numbers.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / numbers.length
+		numbers.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / numbers.length,
 	);
 	const max = Math.max(...numbers);
 	return { avg, stdev, max };

benchmark/bench/cli-startup.js

@@ -1,16 +1,15 @@
 import { fileURLToPath } from 'node:url';
-import { exec } from 'tinyexec';
 import { markdownTable } from 'markdown-table';
+import { exec } from 'tinyexec';
 import { astroBin, calculateStat } from './_util.js';
 
 /** Default project to run for this benchmark if not specified */
 export const defaultProject = 'render-default';
 
 /**
  * @param {URL} projectDir
- * @param {URL} outputFile
  */
-export async function run(projectDir, outputFile) {
+export async function run(projectDir) {
 	const root = fileURLToPath(projectDir);
 
 	console.log('Benchmarking `astro --help`...');
@@ -28,7 +27,7 @@ export async function run(projectDir, outputFile) {
 		printResult({
 			'astro --help': helpStat,
 			'astro info': infoStat,
-		})
+		}),
 	);
 	console.log('='.repeat(10));
 }
@@ -69,6 +68,6 @@ function printResult(result) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 }

benchmark/bench/codspeed.js

@@ -0,0 +1,107 @@
+import fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+import { waitUntilBusy } from 'port-authority';
+import { Bench } from 'tinybench';
+import { exec } from 'tinyexec';
+import { astroBin } from './_util.js';
+import { benchmarkRenderTime } from './render.js';
+import { benchmarkCannon } from './server-stress.js';
+
+export async function run({ memory, render, _stress }) {
+	const bench = process.env.CODSPEED ? withCodSpeed(new Bench()) : new Bench();
+
+	bench
+		.add('memory', async () => {
+			await exec('node', ['--expose-gc', '--max_old_space_size=10000', astroBin, 'build'], {
+				nodeOptions: {
+					cwd: memory.root,
+					stdio: 'inherit',
+					env: {
+						ASTRO_TIMER_PATH: memory.output,
+					},
+				},
+			});
+		})
+		.add(
+			'render',
+			async () => {
+				console.info('Bench rendering.');
+				const port = 4322;
+
+				console.info('Previewing...');
+				const previewProcess = exec(astroBin, ['preview', '--port', port], {
+					nodeOptions: {
+						cwd: render.root,
+						stdio: 'inherit',
+					},
+				});
+
+				console.info('Waiting for server ready...');
+				await waitUntilBusy(port, { timeout: 5000 });
+
+				console.info('Running benchmark...');
+				const results = await benchmarkRenderTime();
+
+				console.info('Killing server...');
+				if (!previewProcess.kill('SIGTERM')) {
+					console.warn('Failed to kill server process id:', previewProcess.pid);
+				}
+
+				return results;
+			},
+			{
+				async beforeAll() {
+					await exec(astroBin, ['build'], {
+						nodeOptions: {
+							cwd: render.root,
+							stdio: 'inherit',
+						},
+					});
+				},
+			},
+		);
+	// NOTE: not needed for now
+	// .add(
+	// 	'stress',
+	// 	async () => {
+	// 		console.info('Bench stress test.');
+	// 		const port = 4323;
+	//
+	// 		console.info('Previewing...');
+	// 		const previewProcess = exec(astroBin, ['preview', '--port', port], {
+	// 			nodeOptions: {
+	// 				cwd: stress.root,
+	// 				stdio: 'inherit',
+	// 			},
+	// 		});
+	//
+	// 		console.info('Waiting for server ready...');
+	// 		await waitUntilBusy(port, { timeout: 5000 });
+	//
+	// 		console.info('Running benchmark...');
+	// 		const result = await benchmarkCannon();
+	//
+	// 		console.info('Killing server...');
+	// 		if (!previewProcess.kill('SIGTERM')) {
+	// 			console.warn('Failed to kill server process id:', previewProcess.pid);
+	// 		}
+	//
+	// 		return result;
+	// 	},
+	// 	{
+	// 		async beforeAll() {
+	// 			console.info('Building...');
+	// 			await exec(astroBin, ['build'], {
+	// 				nodeOptions: {
+	// 					cwd: stress.root,
+	// 					stdio: 'inherit',
+	// 				},
+	// 			});
+	// 		},
+	// 	},
+	// );
+
+	await bench.run();
+	console.table(bench.table());
+}

benchmark/bench/memory.js

@@ -55,6 +55,6 @@ function printResult(output) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 }

benchmark/bench/render.js

@@ -58,7 +58,7 @@ export async function run(projectDir, outputFile) {
 	console.log('Done!');
 }
 
-async function benchmarkRenderTime() {
+export async function benchmarkRenderTime() {
 	/** @type {Record<string, number[]>} */
 	const result = {};
 	for (const fileName of renderPages) {
@@ -95,7 +95,7 @@ function printResult(result) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 }
 

benchmark/bench/server-stress.js

@@ -1,10 +1,10 @@
-import autocannon from 'autocannon';
-import { exec } from 'tinyexec';
-import { markdownTable } from 'markdown-table';
 import fs from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
+import autocannon from 'autocannon';
+import { markdownTable } from 'markdown-table';
 import { waitUntilBusy } from 'port-authority';
 import pb from 'pretty-bytes';
+import { exec } from 'tinyexec';
 import { astroBin } from './_util.js';
 
 const port = 4321;
@@ -27,9 +27,11 @@ export async function run(projectDir, outputFile) {
 	});
 
 	console.log('Previewing...');
-	const previewProcess = execaCommand(`${astroBin} preview --port ${port}`, {
-		cwd: root,
-		stdio: 'inherit',
+	const previewProcess = await exec(astroBin, ['preview', '--port', port], {
+		nodeOptions: {
+			cwd: root,
+			stdio: 'inherit',
+		},
 	});
 
 	console.log('Waiting for server ready...');
@@ -58,7 +60,7 @@ export async function run(projectDir, outputFile) {
 /**
  * @returns {Promise<import('autocannon').Result>}
  */
-async function benchmarkCannon() {
+export async function benchmarkCannon() {
 	return new Promise((resolve, reject) => {
 		const instance = autocannon(
 			{
@@ -75,7 +77,7 @@ async function benchmarkCannon() {
 					instance.stop();
 					resolve(result);
 				}
-			}
+			},
 		);
 		autocannon.track(instance, { renderResultsTable: false });
 	});
@@ -94,7 +96,7 @@ function printResult(output) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 
 	const reqAndBytesTable = markdownTable(
@@ -105,7 +107,7 @@ function printResult(output) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r', 'r'],
-		}
+		},
 	);
 
 	return `${latencyTable}\n\n${reqAndBytesTable}`;

benchmark/index.js

@@ -1,7 +1,7 @@
 import mri from 'mri';
 import fs from 'node:fs/promises';
 import path from 'node:path';
-import { pathToFileURL } from 'node:url';
+import {fileURLToPath, pathToFileURL} from 'node:url';
 
 const args = mri(process.argv.slice(2));
 
@@ -14,6 +14,7 @@ Command
   memory          Run build memory and speed test
   render          Run rendering speed test
   server-stress   Run server stress test
+  codspeed        Run codspeed test
   cli-startup     Run CLI startup speed test
 
 Options
@@ -29,6 +30,7 @@ const benchmarks = {
 	render: () => import('./bench/render.js'),
 	'server-stress': () => import('./bench/server-stress.js'),
 	'cli-startup': () => import('./bench/cli-startup.js'),
+	codspeed: () => import('./bench/codspeed.js')
 };
 
 if (commandName && !(commandName in benchmarks)) {
@@ -37,12 +39,38 @@ if (commandName && !(commandName in benchmarks)) {
 }
 
 if (commandName) {
-	// Run single benchmark
-	const bench = benchmarks[commandName];
-	const benchMod = await bench();
-	const projectDir = await makeProject(args.project || benchMod.defaultProject);
-	const outputFile = await getOutputFile(commandName);
-	await benchMod.run(projectDir, outputFile);
+		if (commandName === 'codspeed') {
+			const memory = await makeProject('memory-default');
+			const render = await makeProject('render-default');
+			const stress = await makeProject('server-stress-default');
+			const rootMemory = fileURLToPath(memory);
+			const rootRender = fileURLToPath(render);
+			const rootStress = fileURLToPath(stress);
+			const bench = benchmarks[commandName];
+			const benchMod = await bench();
+			const payload = {
+				memory: {
+					root: rootMemory,
+					output: await getOutputFile('memory')
+				},
+				render: {
+					root: rootRender,
+					output: await getOutputFile('render')
+				},
+				stress: {
+					root: rootStress,
+					output: await getOutputFile('stress')
+				}
+			};
+			await benchMod.run(payload);
+		} else {
+			// Run single benchmark
+			const bench = benchmarks[commandName];
+			const benchMod = await bench();
+			const projectDir = await makeProject(args.project || benchMod.defaultProject);
+			const outputFile = await getOutputFile(commandName);
+			await benchMod.run(projectDir, outputFile);
+		}
 } else {
 	// Run all benchmarks
 	for (const name in benchmarks) {
@@ -54,7 +82,7 @@ if (commandName) {
 	}
 }
 
-async function makeProject(name) {
+export async function makeProject(name) {
 	console.log('Making project:', name);
 	const projectDir = new URL(`./projects/${name}/`, import.meta.url);
 
@@ -78,6 +106,5 @@ async function getOutputFile(benchmarkName) {
 
 	// Prepare output file directory
 	await fs.mkdir(new URL('./', file), { recursive: true });
-
 	return file;
 }

benchmark/package.json

@@ -18,5 +18,9 @@
     "pretty-bytes": "^6.1.1",
     "sharp": "^0.33.3",
     "tinyexec": "^0.3.1"
+  },
+  "devDependencies": {
+    "@codspeed/tinybench-plugin": "^3.1.0",
+    "tinybench": "^2.6.0"
   }
 }