chore: codspeed benchmark

.github/workflows/continuous_benchmark.yml

@@ -0,0 +1,50 @@
+name: Continuous benchmark
+
+on:
+  workflow_dispatch:
+  pull_request: 
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+env:
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
+  FORCE_COLOR: true
+  CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }}
+  CODSPEED: true
+
+jobs:
+  codspeed:
+    if: ${{ github.repository_owner == 'withastro' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Setup PNPM
+        uses: pnpm/action-setup@v3
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        run: pnpm run build
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@b587655f756aab640e742fec141261bc6f0a569d # v3.0.1
+        timeout-minutes: 30
+        with:
+          run: pnpm benchmark codspeed
+

benchmark/bench/_util.js

@@ -14,7 +14,7 @@ export const astroBin = path.resolve(astroPkgPath, '../astro.js');
 export function calculateStat(numbers) {
 	const avg = numbers.reduce((a, b) => a + b, 0) / numbers.length;
 	const stdev = Math.sqrt(
-		numbers.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / numbers.length
+		numbers.map((x) => Math.pow(x - avg, 2)).reduce((a, b) => a + b, 0) / numbers.length,
 	);
 	const max = Math.max(...numbers);
 	return { avg, stdev, max };

benchmark/bench/cli-startup.js

@@ -1,16 +1,15 @@
 import { fileURLToPath } from 'node:url';
-import { exec } from 'tinyexec';
 import { markdownTable } from 'markdown-table';
+import { exec } from 'tinyexec';
 import { astroBin, calculateStat } from './_util.js';
 
 /** Default project to run for this benchmark if not specified */
 export const defaultProject = 'render-default';
 
 /**
  * @param {URL} projectDir
- * @param {URL} outputFile
  */
-export async function run(projectDir, outputFile) {
+export async function run(projectDir) {
 	const root = fileURLToPath(projectDir);
 
 	console.log('Benchmarking `astro --help`...');
@@ -28,7 +27,7 @@ export async function run(projectDir, outputFile) {
 		printResult({
 			'astro --help': helpStat,
 			'astro info': infoStat,
-		})
+		}),
 	);
 	console.log('='.repeat(10));
 }
@@ -69,6 +68,6 @@ function printResult(result) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 }

benchmark/bench/codspeed.js

@@ -0,0 +1,50 @@
+import path from 'node:path';
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+import { Bench } from 'tinybench';
+import { exec } from 'tinyexec';
+import { renderPages } from '../make-project/render-default.js';
+import { astroBin } from './_util.js';
+
+export async function run({ memory: _memory, render, stress: _stress }) {
+	const options = {
+		iterations: 10,
+	};
+	const bench = process.env.CODSPEED ? withCodSpeed(new Bench(options)) : new Bench(options);
+	let app;
+	bench.add(
+		'Rendering',
+		async () => {
+			console.info('Start task.');
+			const result = {};
+			for (const fileName of renderPages) {
+				const pathname = '/' + fileName.slice(0, -path.extname(fileName).length);
+				const request = new Request(new URL(pathname, 'http://exmpale.com'));
+				const response = await app.render(request);
+				const html = await response.text();
+				if (!result[pathname]) result[pathname] = [];
+				result[pathname].push(html);
+			}
+			console.info('Finish task.');
+			return result;
+		},
+		{
+			async beforeAll() {
+				// build for rendering
+				await exec(astroBin, ['build'], {
+					nodeOptions: {
+						cwd: render.root,
+						stdio: 'inherit',
+					},
+				});
+
+				const entry = new URL('./dist/server/entry.mjs', `file://${render.root}`);
+				const { manifest, createApp } = await import(entry);
+				app = createApp(manifest);
+				app.manifest = manifest;
+			},
+		},
+	);
+
+	await bench.run();
+	console.table(bench.table());
+}

benchmark/bench/memory.js

@@ -55,6 +55,6 @@ function printResult(output) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 }

benchmark/bench/render.js

@@ -1,12 +1,12 @@
-import { exec } from 'tinyexec';
-import { markdownTable } from 'markdown-table';
 import fs from 'node:fs/promises';
 import http from 'node:http';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { markdownTable } from 'markdown-table';
 import { waitUntilBusy } from 'port-authority';
-import { calculateStat, astroBin } from './_util.js';
+import { exec } from 'tinyexec';
 import { renderPages } from '../make-project/render-default.js';
+import { astroBin, calculateStat } from './_util.js';
 
 const port = 4322;
 
@@ -58,14 +58,14 @@ export async function run(projectDir, outputFile) {
 	console.log('Done!');
 }
 
-async function benchmarkRenderTime() {
+export async function benchmarkRenderTime(portToListen = port) {
 	/** @type {Record<string, number[]>} */
 	const result = {};
 	for (const fileName of renderPages) {
 		// Render each file 100 times and push to an array
 		for (let i = 0; i < 100; i++) {
 			const pathname = '/' + fileName.slice(0, -path.extname(fileName).length);
-			const renderTime = await fetchRenderTime(`http://localhost:${port}${pathname}`);
+			const renderTime = await fetchRenderTime(`http://localhost:${portToListen}${pathname}`);
 			if (!result[pathname]) result[pathname] = [];
 			result[pathname].push(renderTime);
 		}
@@ -95,7 +95,7 @@ function printResult(result) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 }
 

benchmark/bench/server-stress.js

@@ -1,10 +1,10 @@
-import autocannon from 'autocannon';
-import { exec } from 'tinyexec';
-import { markdownTable } from 'markdown-table';
 import fs from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
+import autocannon from 'autocannon';
+import { markdownTable } from 'markdown-table';
 import { waitUntilBusy } from 'port-authority';
 import pb from 'pretty-bytes';
+import { exec } from 'tinyexec';
 import { astroBin } from './_util.js';
 
 const port = 4321;
@@ -27,9 +27,11 @@ export async function run(projectDir, outputFile) {
 	});
 
 	console.log('Previewing...');
-	const previewProcess = execaCommand(`${astroBin} preview --port ${port}`, {
-		cwd: root,
-		stdio: 'inherit',
+	const previewProcess = await exec(astroBin, ['preview', '--port', port], {
+		nodeOptions: {
+			cwd: root,
+			stdio: 'inherit',
+		},
 	});
 
 	console.log('Waiting for server ready...');
@@ -58,7 +60,7 @@ export async function run(projectDir, outputFile) {
 /**
  * @returns {Promise<import('autocannon').Result>}
  */
-async function benchmarkCannon() {
+export async function benchmarkCannon() {
 	return new Promise((resolve, reject) => {
 		const instance = autocannon(
 			{
@@ -75,7 +77,7 @@ async function benchmarkCannon() {
 					instance.stop();
 					resolve(result);
 				}
-			}
+			},
 		);
 		autocannon.track(instance, { renderResultsTable: false });
 	});
@@ -94,7 +96,7 @@ function printResult(output) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r'],
-		}
+		},
 	);
 
 	const reqAndBytesTable = markdownTable(
@@ -105,7 +107,7 @@ function printResult(output) {
 		],
 		{
 			align: ['l', 'r', 'r', 'r', 'r'],
-		}
+		},
 	);
 
 	return `${latencyTable}\n\n${reqAndBytesTable}`;

benchmark/index.js

@@ -1,7 +1,7 @@
 import mri from 'mri';
 import fs from 'node:fs/promises';
 import path from 'node:path';
-import { pathToFileURL } from 'node:url';
+import {fileURLToPath, pathToFileURL} from 'node:url';
 
 const args = mri(process.argv.slice(2));
 
@@ -14,6 +14,7 @@ Command
   memory          Run build memory and speed test
   render          Run rendering speed test
   server-stress   Run server stress test
+  codspeed        Run codspeed test
   cli-startup     Run CLI startup speed test
 
 Options
@@ -29,6 +30,7 @@ const benchmarks = {
 	render: () => import('./bench/render.js'),
 	'server-stress': () => import('./bench/server-stress.js'),
 	'cli-startup': () => import('./bench/cli-startup.js'),
+	codspeed: () => import('./bench/codspeed.js')
 };
 
 if (commandName && !(commandName in benchmarks)) {
@@ -37,12 +39,26 @@ if (commandName && !(commandName in benchmarks)) {
 }
 
 if (commandName) {
-	// Run single benchmark
-	const bench = benchmarks[commandName];
-	const benchMod = await bench();
-	const projectDir = await makeProject(args.project || benchMod.defaultProject);
-	const outputFile = await getOutputFile(commandName);
-	await benchMod.run(projectDir, outputFile);
+		if (commandName === 'codspeed') {
+			const render = await makeProject('render-bench');
+			const rootRender = fileURLToPath(render);
+			const bench = benchmarks[commandName];
+			const benchMod = await bench();
+			const payload = {
+				render: {
+					root: rootRender,
+					output: await getOutputFile('render')
+				},
+			};
+			await benchMod.run(payload);
+		} else {
+			// Run single benchmark
+			const bench = benchmarks[commandName];
+			const benchMod = await bench();
+			const projectDir = await makeProject(args.project || benchMod.defaultProject);
+			const outputFile = await getOutputFile(commandName);
+			await benchMod.run(projectDir, outputFile);
+		}
 } else {
 	// Run all benchmarks
 	for (const name in benchmarks) {
@@ -54,7 +70,7 @@ if (commandName) {
 	}
 }
 
-async function makeProject(name) {
+export async function makeProject(name) {
 	console.log('Making project:', name);
 	const projectDir = new URL(`./projects/${name}/`, import.meta.url);
 
@@ -78,6 +94,5 @@ async function getOutputFile(benchmarkName) {
 
 	// Prepare output file directory
 	await fs.mkdir(new URL('./', file), { recursive: true });
-
 	return file;
 }