feat(benchmark): Add benchmark for full language server (#15)

* feat(benchmark): Add benchmark for full language server

* chore: lockfile

* docs: add README

.github/workflows/benchmark.yaml

@@ -37,6 +37,9 @@ jobs:
       - name: Install dependencies
         run: just install
 
+      - name: Build (native)
+        run: just build release
+
       - name: Build (WASM)
         run: just build-wasm benchmark
 
@@ -46,5 +49,5 @@ jobs:
       - name: Run the benchmarks
         uses: CodSpeedHQ/action@v3
         with:
-          run: cargo codspeed run && pnpm -C ./packages/benchmark-wasm run benchmark-codspeed
+          run: cargo codspeed run && pnpm -C ./packages/benchmark-wasm run benchmark-codspeed && pnpm -C ./packages/language-server-tests-benchmarks run benchmark-codspeed
           token: ${{ secrets.CODSPEED_TOKEN }}

justfile

@@ -33,3 +33,5 @@ benchmark:
 	cargo bench
 	echo "Running WASM benchmarks..."
 	pnpm -C ./packages/benchmark-wasm run benchmark --run
+	echo "Running Language Server benchmarks..."
+	pnpm -C ./packages/language-server-tests-benchmarks run benchmark --run

packages/language-server-tests-benchmarks/README.md

@@ -0,0 +1,9 @@
+# language-server-tests-benchmarks
+
+The tests and benchmarks in this folder aims to test and benchmark the language server in a close-to-reality scenario, using the same JavaScript client powered by `vscode-jsonrpc` used in VS Code.
+
+## Why are the benchmarks so slow?
+
+If you're used to the numbers from the benchmarks of the language services, the numbers here might be surprisingly slow. The reason for that is that the benchmark also includes the large overhead caused by the client-server communication. Even if the language server can sometimes answer in 50-100μs, just sending and waiting for the response can take 95-99% of the time the benchmark measures, leading to times closer to 1-2ms.
+
+Unfortunately, due to the multiple processes involved it's not possible to get accurate flamegraphs from CodSpeed for these benchmarks at the time of writing. Locally, [flamegraph](https://github.com/flamegraph-rs/flamegraph) can be used, but it requires a bit of setup, especially on non-Linux systems.

packages/language-server-tests-benchmarks/fixture/colors_benchmark.css

@@ -0,0 +1,17 @@
+.header {
+    background-color: #333;
+    color: white;
+    padding: 15px 20px;
+    text-align: center;
+}
+
+h1 {
+    color: red;
+    background-color: lab(50% 50% 50%);
+}
+
+.header .logo {
+    font-size: 2rem;
+    font-weight: bold;
+    color: lch(50% 50% 50%);
+}

packages/language-server-tests-benchmarks/fixture/folding_benchmark.css

@@ -0,0 +1,74 @@
+/* General Layout */
+.container {
+    width: 100%;
+    padding: 20px;
+    margin: 0 auto;
+    max-width: 1200px;
+}
+
+.header {
+    background-color: #333;
+    color: white;
+    padding: 15px 20px;
+    text-align: center;
+}
+
+.header .logo {
+    font-size: 2rem;
+    font-weight: bold;
+}
+
+.sidebar {
+    flex: 1;
+    padding: 20px;
+    background-color: #fff;
+    border-radius: 8px;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+.footer {
+    margin-top: 40px;
+    padding: 20px;
+    background-color: #222;
+    color: white;
+    text-align: center;
+    font-size: 0.9rem;
+}
+
+/* Media Queries */
+@media (max-width: 1024px) {
+    .content {
+        flex-direction: column;
+        align-items: center;
+    }
+    .main,
+    .sidebar {
+        width: 80%;
+        margin-bottom: 20px;
+    }
+}
+
+@media (max-width: 768px) {
+    .header .logo {
+        font-size: 1.5rem;
+    }
+    .footer {
+        font-size: 0.8rem;
+    }
+}
+
+@media (max-width: 480px) {
+    .container {
+        padding: 10px;
+    }
+    .header {
+        padding: 10px;
+    }
+    .content {
+        flex-direction: column;
+    }
+    .main,
+    .sidebar {
+        padding: 10px;
+    }
+}

packages/language-server-tests-benchmarks/fixture/hover_benchmark.css

@@ -0,0 +1,3 @@
+h1 {
+    color: red;
+}

packages/language-server-tests/package.json → packages/language-server-tests-benchmarks/package.json

@@ -1,13 +1,17 @@
 {
-	"name": "language-server-tests",
+	"name": "language-server-tests-benchmarks",
 	"type": "module",
 	"private": true,
 	"scripts": {
-		"test": "vitest"
+		"test": "vitest",
+		"benchmark": "vitest bench -c vitest.config.bench.ts",
+		"benchmark-codspeed": "CODSPEED=true pnpm run benchmark"
 	},
 	"dependencies": {
+		"@codspeed/vitest-plugin": "^3.1.1",
 		"@types/node": "^22.10.1",
 		"vitest": "^2.1.8",
+		"vscode-langservers-extracted": "^4.10.0",
 		"vscode-languageclient": "^9.0.1",
 		"vscode-languageserver-protocol": "^3.17.5",
 		"vscode-languageserver-textdocument": "^1.0.12",

packages/language-server-tests-benchmarks/src/benchmarks/css/colors.bench.ts

@@ -0,0 +1,54 @@
+import { afterAll, bench, describe } from "vitest";
+import { startLanguageServer } from "../../server";
+import { fileURLToPath } from "url";
+
+const filePath = fileURLToPath(
+	new URL("../../../fixture/colors_benchmark.css", import.meta.url)
+);
+
+const weblsp = await startLanguageServer(undefined, "weblsp");
+const weblspUri = (await weblsp.openTextDocument(filePath, "css")).uri;
+const weblspColors = await weblsp.sendDocumentColorRequest(weblspUri);
+
+const vscodeLsp = await startLanguageServer(undefined, "vscode-css");
+const vscodeLspUri = (await vscodeLsp.openTextDocument(filePath, "css")).uri;
+const vscodeColors = await vscodeLsp.sendDocumentColorRequest(vscodeLspUri);
+
+describe("Document Colors", async () => {
+	bench("weblsp - Document Colors", async () => {
+		await weblsp.sendDocumentColorRequest(weblspUri);
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Document Colors", async () => {
+			await vscodeLsp.sendDocumentColorRequest(vscodeLspUri);
+		});
+	}
+});
+
+describe("Color Presentations", async () => {
+	bench("weblsp - Color Presentation", async () => {
+		await weblsp.sendColorPresentationRequest(
+			weblspUri,
+			weblspColors[0].color,
+			weblspColors[0].range
+		);
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Color Presentation", async () => {
+			await vscodeLsp.sendColorPresentationRequest(
+				vscodeLspUri,
+				vscodeColors[0].color,
+				vscodeColors[0].range
+			);
+		});
+	}
+
+	afterAll(async () => {
+		await weblsp.shutdown();
+		await vscodeLsp.shutdown();
+		await weblsp.exit();
+		await vscodeLsp.exit();
+	});
+});

packages/language-server-tests-benchmarks/src/benchmarks/css/folding.bench.ts

@@ -0,0 +1,32 @@
+import { afterAll, bench, describe } from "vitest";
+import { startLanguageServer } from "../../server";
+import { fileURLToPath } from "url";
+
+const filePath = fileURLToPath(
+	new URL("../../../fixture/folding_benchmark.css", import.meta.url)
+);
+
+const weblsp = await startLanguageServer(undefined, "weblsp");
+const weblspUri = (await weblsp.openTextDocument(filePath, "css")).uri;
+
+const vscodeLsp = await startLanguageServer(undefined, "vscode-css");
+const vscodeLspUri = (await vscodeLsp.openTextDocument(filePath, "css")).uri;
+
+describe("Folding Ranges", async () => {
+	bench("weblsp - Folding Ranges", async () => {
+		await weblsp.sendFoldingRangesRequest(weblspUri);
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Folding Ranges", async () => {
+			await vscodeLsp.sendFoldingRangesRequest(vscodeLspUri);
+		});
+	}
+
+	afterAll(async () => {
+		await weblsp.shutdown();
+		await vscodeLsp.shutdown();
+		await weblsp.exit();
+		await vscodeLsp.exit();
+	});
+});

packages/language-server-tests-benchmarks/src/benchmarks/css/hover.bench.ts

@@ -0,0 +1,38 @@
+import { afterAll, bench, describe } from "vitest";
+import { startLanguageServer } from "../../server";
+import { fileURLToPath } from "url";
+
+const filePath = fileURLToPath(
+	new URL("../../../fixture/hover_benchmark.css", import.meta.url)
+);
+
+const weblsp = await startLanguageServer(undefined, "weblsp");
+const weblspUri = (await weblsp.openTextDocument(filePath, "css")).uri;
+
+const vscodeLsp = await startLanguageServer(undefined, "vscode-css");
+const vscodeLspUri = (await vscodeLsp.openTextDocument(filePath, "css")).uri;
+
+describe("Hover", async () => {
+	bench("weblsp - Hover", async () => {
+		await weblsp.sendHoverRequest(weblspUri, {
+			line: 1,
+			character: 6,
+		});
+	});
+
+	if (!process.env.CODSPEED) {
+		bench("vscode-css-languageserver - Hover", async () => {
+			await vscodeLsp.sendHoverRequest(vscodeLspUri, {
+				line: 1,
+				character: 6,
+			});
+		});
+	}
+
+	afterAll(async () => {
+		await weblsp.shutdown();
+		await vscodeLsp.shutdown();
+		await weblsp.exit();
+		await vscodeLsp.exit();
+	});
+});

packages/language-server-tests/tests/server.ts → packages/language-server-tests-benchmarks/src/server.ts

@@ -8,35 +8,61 @@ import { TextDocument } from "vscode-languageserver-textdocument";
 import { URI } from "vscode-uri";
 import * as assert from "node:assert/strict";
 import { fileURLToPath } from "node:url";
-import { createHash, randomBytes } from "node:crypto";
+import { randomBytes } from "node:crypto";
 
-const pathToBinary = fileURLToPath(
-	new URL("../../../target/debug/weblsp", import.meta.url)
-);
+let pathToBinary: string;
+if (process.env.BENCHMARK === "true" || process.env.RELEASE === "true") {
+	pathToBinary = fileURLToPath(
+		new URL("../../../target/release/weblsp", import.meta.url)
+	);
+} else {
+	pathToBinary = fileURLToPath(
+		new URL("../../../target/debug/weblsp", import.meta.url)
+	);
+}
 
 export const fixtureDir = URI.file(
 	fileURLToPath(new URL("./fixture", import.meta.url))
 ).toString();
 
-export type LanguageServerHandle = ReturnType<typeof startLanguageServer>;
+export type LanguageServerHandle = Awaited<
+	ReturnType<typeof startLanguageServer>
+>;
 
-export async function startLanguageServer(cwd?: string | undefined) {
-	console.info(`Starting language server at ${pathToBinary}`);
-	const childProcess = cp.spawn(pathToBinary, [], {
-		env: process.env,
-		cwd,
-		stdio: "pipe",
-	});
+export async function startLanguageServer(
+	cwd?: string | undefined,
+	which: "weblsp" | "vscode-css" = "weblsp"
+) {
+	if (which === "weblsp")
+		console.info(`Starting language server at ${pathToBinary}`);
+
+	const childProcess =
+		which === "weblsp"
+			? cp.spawn(pathToBinary, [], {
+					env: process.env,
+					cwd,
+					stdio: "pipe",
+			  })
+			: cp.fork(
+					"node_modules/vscode-langservers-extracted/bin/vscode-css-language-server",
+					["--stdio", `--clientProcessId=${process.pid.toString()}`],
+					{
+						execArgv: ["--nolazy"],
+						env: process.env,
+						cwd,
+						stdio: "pipe",
+					}
+			  );
 
 	if (!childProcess.stdout || !childProcess.stdin) {
 		throw new Error("Bad stdio configuration, should be pipe");
 	}
 
-	if (process.env.DEBUG) {
-		childProcess.stderr?.on("data", (data) => {
+	childProcess.stderr?.on("data", (data) => {
+		if (process.env.DEBUG) {
 			console.error(data.toString());
-		});
-	}
+		}
+	});
 
 	const connection = _.createProtocolConnection(
 		childProcess.stdout,
@@ -48,7 +74,7 @@ export async function startLanguageServer(cwd?: string | undefined) {
 	connection.onClose((e) => console.log("Closed", e));
 
 	connection.onUnhandledNotification((e) =>
-		console.log("Unhandled notificaiton", e)
+		console.log("Unhandled notification", e)
 	);
 
 	connection.onError((e) => console.log("Error:", e));
@@ -149,6 +175,15 @@ export async function startLanguageServer(cwd?: string | undefined) {
 		}
 	);
 
+	// VS Code's CSS language server crashes if this is not set
+	if (which === "vscode-css") {
+		Object.assign(settings, { "css.lint.validProperties": [] });
+		await connection.sendNotification(
+			_.DidChangeConfigurationNotification.type,
+			{ settings } satisfies _.DidChangeConfigurationParams
+		);
+	}
+
 	return {
 		process: childProcess,
 		connection,

packages/language-server-tests-benchmarks/src/tests/setup.ts

@@ -0,0 +1,9 @@
+import { startLanguageServer } from "../server";
+
+declare global {
+	var languageServer: import("../server").LanguageServerHandle;
+}
+
+if (!globalThis.languageServer) {
+	globalThis.languageServer = await startLanguageServer();
+}

packages/language-server-tests/tests/shutdown.test.ts → packages/language-server-tests-benchmarks/src/tests/shutdown.test.ts

@@ -1,5 +1,4 @@
 import { describe, expect, it } from "vitest";
-import { ServerCapabilities } from "vscode-languageserver-protocol/node";
 
 describe("Language server initilization", () => {
 	it("Can shutdown server", async () => {