Merge pull request #57 from dddsw/updateScreenshotTests

Update playwright tests

.gitignore

@@ -10,3 +10,4 @@ node_modules
 .output
 .idea
 /test-results
+playwright-report

.husky/pre-commit

@@ -1 +1,2 @@
 npx lint-staged
+npm run test

README.md

@@ -6,18 +6,18 @@ This is currently a WIP. Please feel free to raise any issues that you come acro
 
 1. Install dependencies:
 
-```bash
-npm install
-```
+   ```bash
+   npm install
+   ```
 
 2. Start the app:
 
-```bash
-npm run dev
+   ```bash
+   npm run dev
 
-# or start the server and open the app in a new browser tab
-npm run dev -- --open
-```
+   # or start the server and open the app in a new browser tab
+   npm run dev -- --open
+   ```
 
 ## Running the tests
 
@@ -38,7 +38,11 @@ npm run test:ui
 
 You can also run them via the Playwright VS Code extension.
 
-The tests generate page previews in different device sizes. The screenshots are saved under `tests\page-previews`.
+### Visual comparison tests
+
+The tests generate page previews in different device sizes and test to ensure that they haven't changed. The screenshots are saved under `tests\__screenshots__`. and can be regenerated with `npm run test:update-screenshots`.
+
+If a visual test fails, look under the attachments in the report for a visual diff.
 
 ## Contributing
 
@@ -77,3 +81,20 @@ Background colour classes to set the sections to the predefined colours:
 	</div>
 </div>
 ```
+
+### Tests
+
+When adding a new page, add the name and route to `tests\all-pages-to-test.ts` to enable automated accessibility and visual comparison tests.
+
+If adding or changing a link in the nav, ensure it is add/changed in `tests\navigation.spec.ts`.
+
+### Pre-commit hooks
+
+> [!IMPORTANT]
+> Playwright tests are run on pre-commit hooks and will fail if Playwright is not installed. See [running the tests](#running-the-tests) for info on how to set up Playwright
+
+Pre-commit hooks are managed by [Husky](https://typicode.github.io/husky/get-started.html) and will be run whenever you make a commit. If they fail then the commit will be aborted and you will see an error.
+
+To run pre-commit hooks without committing, stage all your changes and then execute the `.husky/pre-commit` file.
+
+To commit but skip pre-commit hooks, run `git commit -m "my amazing commit message" --no-verify`

package.json

@@ -6,9 +6,9 @@
 		"build": "vite build",
 		"package": "svelte-kit package",
 		"preview": "vite preview",
-		"test": "playwright test --retries=3",
+		"test": "playwright test",
 		"test:ui": "playwright test --ui",
-		"test:page-previews": "playwright test tests/page-previews.spec.ts",
+		"test:update-screenshots": "playwright test tests/visual-comparison.spec.ts --update-snapshots",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
 		"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
 		"lint": "prettier --check . & eslint .",

playwright.config.ts

@@ -1,6 +1,6 @@
-import type { PlaywrightTestConfig } from '@playwright/test';
+import { defineConfig } from '@playwright/test';
 
-const config: PlaywrightTestConfig = {
+export default defineConfig({
 	webServer: {
 		command: 'npm run dev',
 		url: 'http://127.0.0.1:5173',
@@ -9,7 +9,8 @@ const config: PlaywrightTestConfig = {
 	use: {
 		baseURL: 'http://127.0.0.1:5173/'
 	},
-	fullyParallel: true
-};
-
-export default config;
+	fullyParallel: true,
+	reporter: [['list'], ['html']],
+	testDir: './tests',
+	snapshotPathTemplate: '{testDir}/__screenshots__/{arg}{ext}'
+});

tests/a11y.spec.ts

@@ -1,5 +1,7 @@
 import { test, devices, expect } from '@playwright/test';
 import AxeBuilder from '@axe-core/playwright';
+import { devicesToTest } from './all-devices-to-test';
+import { pagesToTest } from './all-pages-to-test';
 
 let accessibilityScanner: AxeBuilder;
 
@@ -12,28 +14,12 @@ test.beforeEach(({ page }) => {
 	]);
 });
 
-const viewPorts = new Map([
-	['mobile', 'iPhone SE'],
-	['tablet', 'iPad Mini'],
-	['desktop', 'Desktop Chrome']
-]);
-for (const [viewportName, viewportDevice] of viewPorts) {
-	//Home page
-	test(`Home page a11y checks - ${viewportName}`, async ({ page }) => {
-		await page.setViewportSize(devices[viewportDevice].viewport);
+for (const [pageName, path] of pagesToTest) {
+	for (const [deviceName, deviceViewport] of devicesToTest) {
+		test(`${pageName} page a11y checks on ${deviceName}}`, async ({ page }) => {
+			await page.setViewportSize(devices[deviceViewport].viewport);
 
-		await page.goto('/');
-
-		await reportAccessibilityViolations();
-	});
-
-	// Content pages
-	const routes = ['about', 'sponsorship', '2024', 'code-of-conduct', 'new-speakers-workshop'];
-	for (const route of routes) {
-		test(`${route} page a11y checks - ${viewportName}`, async ({ page }) => {
-			await page.setViewportSize(devices[viewportDevice].viewport);
-
-			await page.goto(route);
+			await page.goto(path);
 
 			await reportAccessibilityViolations();
 		});
@@ -42,22 +28,22 @@ for (const [viewportName, viewportDevice] of viewPorts) {
 
 async function reportAccessibilityViolations() {
 	// Scan the page with axe to look for a11y violations
-	let violations = (await accessibilityScanner.analyze()).violations;
+	const violations = (await accessibilityScanner.analyze()).violations;
 
 	// Make Playwright output the key information as a (more) human readable string
 	let violationDescription = '';
 	if (violations.length > 0) {
 		violationDescription = `Warning - ${violations.length} a11y rule violations found\n\n`;
 
 		for (let i = 0; i < violations.length; i++) {
-			let violation = violations[i];
+			const violation = violations[i];
 			violationDescription += `########################################################\n`;
 			violationDescription += `Rule Violation ${i + 1}\n`;
 			violationDescription += `${violation.impact?.toUpperCase()} - ${violation.help}\n`;
 			violationDescription += `${violation.helpUrl}\n`;
 
 			for (let j = 0; j < violation.nodes.length; j++) {
-				let instance = violation.nodes[j];
+				const instance = violation.nodes[j];
 
 				violationDescription += `--------------------------------------------------------\n`;
 				violationDescription += `Instance ${j + 1}/${violation.nodes.length}\n\n`;

tests/all-devices-to-test.ts

@@ -0,0 +1,5 @@
+export const devicesToTest: [deviceName: string, deviceViewport: string][] = [
+	['desktop', 'Desktop Chrome'],
+	['tablet', 'iPad Mini'],
+	['mobile', 'iPhone SE']
+];

tests/all-pages-to-test.ts

@@ -0,0 +1,9 @@
+export const pagesToTest: [pageName: string, path: string][] = [
+	['Home', '/'],
+	['About', 'about'],
+	['Sponsorship', 'sponsorship'],
+	['Venue', 'venue'],
+	['Code of conduct', 'code-of-conduct'],
+	['Code of conduct (internal)', 'code-of-conduct/internal'],
+	['New speakers workshop', 'new-speakers-workshop']
+];

tests/navbar.spec.ts

@@ -1,96 +1,85 @@
 import { test, expect, devices } from '@playwright/test';
+import { Header } from './page-object/header';
 
-test('Clicking nav button displays nav links dropdown', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+test.describe('Navbar', () => {
+	let header: Header;
 
-	// Click nav drop down
-	await page.locator('.main-nav-button').click();
+	test.beforeEach(async ({ page }) => {
+		// Go to about page (to avoid carousel weirdness)
+		await page.goto('/about');
 
-	await expect(page.locator('.nav-link-container')).toBeVisible();
-});
+		//Ensure everything has loaded to prevent flakiness
+		await page.waitForLoadState('networkidle');
 
-test('Clicking nav button closes nav links dropdown', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+		header = new Header(page);
+	});
 
-	// Click nav drop down
-	await page.locator('.main-nav-button').click();
+	test('Clicking nav button displays nav links dropdown', async () => {
+		// Click nav drop down
+		await header.navDropDownButton.click();
 
-	// Click nav drop down again
-	await page.locator('.main-nav-button').click();
+		await expect(header.navDropDown).toBeVisible();
+	});
 
-	await expect(page.locator('.nav-link-container')).toBeHidden();
-});
+	test('Clicking nav button closes nav links dropdown', async () => {
+		// Click nav drop down
+		await header.navDropDownButton.click();
 
-test('Clicking nav link closes nav links dropdown', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+		// Click nav drop down again
+		await header.navDropDownButton.click();
 
-	// Click nav drop down
-	await page.locator('.main-nav-button').click();
+		await expect(header.navDropDown).toBeHidden();
+	});
 
-	// Click a nav link
-	await page.locator('.nav-link-container a[href="/sponsorship"]').click();
+	test('Clicking nav link closes nav links dropdown', async () => {
+		// Click nav drop down
+		await header.navDropDownButton.click();
 
-	await expect(page.locator('.nav-link-container')).toBeHidden();
-});
+		// Click a nav link
+		await header.getNavLinkTo('sponsorship').click();
 
-test('Clicking logo closes nav links dropdown', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+		await expect(header.navDropDown).toBeHidden();
+	});
 
-	// Click nav drop down
-	await page.locator('.main-nav-button').click();
+	test('Clicking logo closes nav links dropdown', async () => {
+		// Click nav drop down
+		await header.navDropDownButton.click();
 
-	// Click logo
-	await page.locator('img[class~="logo"]').click();
+		// Click logo
+		await header.logo.click();
 
-	await expect(page.locator('.nav-link-container')).toBeHidden();
-});
+		await expect(header.navDropDown).toBeHidden();
+	});
 
-test('Clicking logo does not open nav links dropdown', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+	test('Clicking logo does not open nav links dropdown', async () => {
+		// Click logo
+		await header.logo.click();
 
-	// Click logo
-	await page.locator('img[class~="logo"]').click();
+		await expect(header.navDropDown).toBeHidden();
+	});
 
-	await expect(page.locator('.nav-link-container')).toBeHidden();
-});
-
-test('Displays down arrow icon in unexpanded state', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+	test('Displays down arrow icon in unexpanded state', async () => {
+		await expect(header.navDropDownButton).toContainText('expand_more');
+	});
 
-	await expect(page.locator('.main-nav-button span.icon')).toContainText('expand_more');
-});
+	test('Displays up arrow icon in expanded state', async () => {
+		// Click nav drop down
+		await header.navDropDownButton.click();
 
-test('Displays up arrow icon in expanded state', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+		await expect(header.navDropDownButton).toContainText('expand_less');
+	});
 
-	// Click nav drop down
-	await page.locator('.main-nav-button').click();
-
-	await expect(page.locator('.main-nav-button span.icon')).toContainText('expand_less');
-});
-
-test('Does not show current route in nav links drop down', async ({ page }) => {
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
-
-	// Click nav drop down
-	await page.locator('.main-nav-button').click();
-
-	await expect(page.locator('.nav-link-container a[href="/about"]')).toBeHidden();
-});
+	test('Does not show current route in nav links drop down', async () => {
+		// Click nav drop down
+		await header.navDropDownButton.click();
 
-test('Does not show button text in smaller screens', async ({ page }) => {
-	await page.setViewportSize(devices['iPhone SE'].viewport);
+		await expect(header.getNavLinkTo('about')).toBeHidden();
+	});
 
-	// Go to about page (to avoid carousel weirdness)
-	await page.goto('/about');
+	test('Does not show button text in smaller screens', async ({ page }) => {
+		// Set to mobile view
+		await page.setViewportSize(devices['iPhone SE'].viewport);
 
-	await expect(page.locator('.nav-button-text')).toBeHidden();
+		await expect(page.locator('.nav-button-text')).toBeHidden();
+	});
 });