Improve messaging

actions/ui-check/package.json

@@ -9,10 +9,10 @@
 		"test:e2e:interactive": "PUPPETEER_HEADLESS=false wp-scripts test-e2e --config tests/e2e/jest.config.js",
 		"test:e2e": "wp-scripts test-e2e --config tests/e2e/jest.config.js",
 		"test:e2e:dev": "DEV_MODE=true wp-scripts test-e2e --config tests/e2e/jest.config.js",
-		"test:e2e:sanity": "SANITY_MODE=true DEV_MODE=true wp-scripts test-e2e --config tests/e2e/jest.sanity.js",
+		"test:e2e:sanity": "SANITY_MODE=true wp-scripts test-e2e --config tests/e2e/jest.sanity.js",
 		"test:unit": "wp-scripts test-unit-js --config tests/unit/jest.config.js --passWithNoTests",
 		"test:unit:dev": "DEV_MODE=true wp-scripts test-unit-js --config tests/unit/jest.config.js",
-		"test:unit:sanity": "SANITY_MODE=true DEV_MODE=true wp-scripts test-e2e --config tests/unit/jest.sanity.js",
+		"test:unit:sanity": "SANITY_MODE=true wp-scripts test-e2e --config tests/unit/jest.sanity.js",
 		"start": "npm run install:environment && npm run test:unit && npm run test:e2e"
 	},
 	"author": "The WordPress Contributors",

actions/ui-check/tests/e2e/jest.config.js

@@ -19,8 +19,13 @@ let configs = {
 		// Exclude any sanity tests
 		'!**/e2e/**/sanity/**',
 	],
+	reporters: [ '<rootDir>/../reporters/index.js' ],
 };
 
+if ( process.env.DEV_MODE ) {
+	configs.reporters = [ 'default' ];
+}
+
 // When run using NPX there are issues related to running in a `node_modules` folder.
 if (
 	process.env.INIT_CWD &&

actions/ui-check/tests/e2e/specs/a11y/index.test.js

@@ -4,12 +4,23 @@
 import urls from './pages';
 import {
 	createURL,
-	cleanErrorMessage,
 	printMessage,
 	getEnvironmentVariable,
 } from '../../../utils';
 
-describe( 'Accessibility', () => {
+/**
+ * Removes some noise that exists in the testing framework error messages.
+ * @param {string} msg Error message thrown by testing framework.
+ * @returns {string}
+ */
+export const cleanErrorMessage = ( msg ) => {
+	return msg
+		.replace( 'expect(received).toPassAxeTests(expected)', '' )
+		.replace( 'Expected page to pass Axe accessibility tests.', '' )
+		.replace( /^\s*$(?:\r\n?|\n)/, '\n' );
+};
+
+describe.each( urls, () => {
 	const envVar = getEnvironmentVariable( process.env.TEST_ACCESSIBILITY );
 	const testAccessibility = envVar === 'true';
 	const accessibilityTest = testAccessibility ? 'wcag2a' : 'best-practice';
@@ -18,27 +29,24 @@ describe( 'Accessibility', () => {
 	// Temporarily set everything as a warning.
 	const noticeType = 'warnings';
 
-	test.each( urls )(
-		`Should pass ${ accessibilityTest } Axe tests on %s`,
-		async ( name, path, query ) => {
-			await page.goto( createURL( path, query ) );
+	test( `Should pass ${ accessibilityTest } Axe tests on %s`, async ( name, path, query ) => {
+		await page.goto( createURL( path, query ) );
 
-			try {
-				await expect( page ).toPassAxeTests( {
-					options: {
-						runOnly: {
-							type: 'tag',
-							values: [ accessibilityTest ],
-						},
+		try {
+			await expect( page ).toPassAxeTests( {
+				options: {
+					runOnly: {
+						type: 'tag',
+						values: [ accessibilityTest ],
 					},
-					exclude: [ [ '.entry-content' ] ],
-				} );
-			} catch ( e ) {
-				printMessage( noticeType, [
-					`Running tests on ${ name } ${ path }${ query } using: \nhttps://github.com/wpaccessibility/a11y-theme-unit-test`,
-					cleanErrorMessage( e.message ),
-				] );
-			}
+				},
+				exclude: [ [ '.entry-content' ] ],
+			} );
+		} catch ( e ) {
+			printMessage( noticeType, [
+				`Running tests on ${ name } ${ path }${ query } using: \nhttps://github.com/wpaccessibility/a11y-theme-unit-test`,
+				cleanErrorMessage( e.message ),
+			] );
 		}
-	);
+	} );
 } );

actions/ui-check/tests/e2e/specs/page/body-class/index.js

@@ -12,8 +12,7 @@ export default async ( url, bodyClass ) => {
 	const bodyClassName = await getElementPropertyAsync( body, 'className' );
 
 	return errorWithMessageOnFail(
-		`${ url } does not contain a body class`,
-		'page-should-contain-body-class',
+		`Unable to find body class when loading a page using ${ url }`,
 		() => {
 			expect( bodyClassName.split( ' ' ) ).toContain( bodyClass );
 		}

actions/ui-check/tests/e2e/specs/page/complete-output/index.js

@@ -5,8 +5,7 @@ import { errorWithMessageOnFail } from '../../../../utils';
 
 export default async ( url, responseText ) => {
 	return errorWithMessageOnFail(
-		`${ url } contains incomplete output. Make sure the page contains valid html.`,
-		'page-should-have-complete-output',
+		`Loading a page using ${ url } contains incomplete output. Make sure the page contains valid html.`,
 		() => {
 			expect( responseText ).toMatch( /<\/(html|rss)>\s*$/ );
 		}

actions/ui-check/tests/e2e/specs/page/frontpage-template/index.js

@@ -1,22 +1,20 @@
 /**
  * Internal dependencies
  */
-import {
-	errorWithMessageOnFail,
-	getFileNameFromPath,
-} from '../../../../utils';
+import { errorWithMessageOnFail, getFileNameFromPath } from '../../../../utils';
 
 export default async ( url ) => {
+	let template, filename;
 
-	const template = await page.$eval( '#template', (el) => el.value);
-	const filename = getFileNameFromPath( template );
+	try {
+		template = await page.$eval( '#template', ( el ) => el.value );
+		filename = getFileNameFromPath( template );
+	} catch ( ex ) {}
 
 	return errorWithMessageOnFail(
-		`${url} Frontpage template file must not be page.php`,
-		'frontpage-should-show-correct-content',
+		`Frontpage template file must not be page.php`,
 		() => {
 			expect( filename ).not.toBe( 'page.php' );
-		},
+		}
 	);
-
 };

actions/ui-check/tests/e2e/specs/page/frontpage/index.js

@@ -12,11 +12,9 @@ export default async ( url ) => {
 	const bodyClassName = await getElementPropertyAsync( body, 'className' );
 
 	return errorWithMessageOnFail(
-		`${url} does not contain the blog body class`,
-		'frontpage-should-show-correct-content',
+		`${ url } does not contain the blog body class`,
 		() => {
 			expect( bodyClassName.split( ' ' ) ).toContain( 'blog' );
-		},
+		}
 	);
-
 };

actions/ui-check/tests/e2e/specs/page/frontpage/sanity/index.test.js

@@ -19,9 +19,10 @@ describe( 'Sanity: Blog Body Class Test', () => {
 	} );
 
 	it( 'Page should FAIL when response is empty', async () => {
-		await page.goto( `file:${ path.join( __dirname, 'html/empty.html' ) }` );
+		await page.goto(
+			`file:${ path.join( __dirname, 'html/empty.html' ) }`
+		);
 
 		expect( await test( '', 'frontpage' ) ).toBeFalsy();
 	} );
-
 } );

actions/ui-check/tests/e2e/specs/page/index.test.js

@@ -6,7 +6,12 @@ const fetch = require( 'node-fetch' );
 /**
  * Internal dependencies
  */
-import { getFileNameFromPath, getTestUrls, goTo, getSiteInfo } from '../../../utils';
+import {
+	getFileNameFromPath,
+	getTestUrls,
+	goTo,
+	getSiteInfo,
+} from '../../../utils';
 
 import bodyClassTest from './body-class';
 import phpErrorsTest from './php-errors';
@@ -23,11 +28,11 @@ let urls = [ [ '/', '?feed=rss2', '' ], ...getTestUrls() ];
 
 const getUrlPathWithTemplate = async ( urlPath ) => {
 	const template = await page.$eval( '#template', ( el ) => el.value );
-	return `${ urlPath } (via: ${ getFileNameFromPath( template ) })`;
+	return getFileNameFromPath( template );
 };
 
 // Some basic tests that apply to every page
-describe.each( urls )( 'Test URL %s%s', ( url, queryString, bodyClass ) => {
+describe.each( urls )( '%s%s', ( url, queryString, bodyClass ) => {
 	let pageResponse, urlPath;
 
 	beforeAll( async () => {
@@ -39,7 +44,7 @@ describe.each( urls )( 'Test URL %s%s', ( url, queryString, bodyClass ) => {
 		} catch ( ex ) {}
 	} );
 
-	it( 'Page should contain body class ' + bodyClass, async () => {
+	it( 'Page should contain body class', async () => {
 		// Make sure the page content appears to be appropriate for the URL.
 		await bodyClassTest( urlPath, bodyClass );
 	} );
@@ -68,13 +73,12 @@ describe.each( urls )( 'Test URL %s%s', ( url, queryString, bodyClass ) => {
 		// See https://make.wordpress.org/themes/handbook/review/required/#selling-credits-and-links
 		await unexpectedLinksTest( urlPath );
 	} );
-
 } );
 
 // Some basic tests that apply only to the frontpage
 let homeurl = [ [ '/', '', 'home' ] ];
 
-describe.each( homeurl )( 'Test URL %s%s', ( url, queryString ) => {
+describe.each( homeurl )( '%s%s', ( url, queryString ) => {
 	let pageResponse, urlPath;
 
 	beforeAll( async () => {
@@ -93,25 +97,23 @@ describe.each( homeurl )( 'Test URL %s%s', ( url, queryString ) => {
 	it( 'Frontpage template should not be page.php', async () => {
 		await frontpageTemplateTest( urlPath );
 	} );
-
 } );
 
 // Test if theme and author URI have a valid responses
 const siteInfo = getSiteInfo();
-let theme_urls = [...siteInfo.theme_urls];
+let theme_urls = [ ...siteInfo.theme_urls ];
 
-if ( theme_urls[0] ) {
-	describe.each( theme_urls )('Test URL %s', ( url ) => {
+if ( theme_urls[ 0 ] ) {
+	describe.each( theme_urls )( '%s', ( url ) => {
 		let pageResponse;
 
-		beforeAll(async () => {
+		beforeAll( async () => {
 			pageResponse = await page.goto( url );
-		});
+		} );
 
 		it( 'Page should return 200 status', async () => {
 			const status = await pageResponse.status();
 			await pageStatusTest( url, status );
-		});
-
-	});
+		} );
+	} );
 }

actions/ui-check/tests/e2e/specs/page/js-errors/index.js

@@ -14,10 +14,9 @@ page.on( 'pageerror', ( error ) => {
 
 export default async ( url ) => {
 	return errorWithMessageOnFail(
-		`${ url } contains javascript errors. Found ${ removeLocalPathRefs(
+		`Loading the page using ${ url } contains javascript errors. Found ${ removeLocalPathRefs(
 			jsError
 		) }`,
-		'browser-console-should-not-contain-errors',
 		() => {
 			expect( jsError ).toBeFalsy();
 		}

actions/ui-check/tests/e2e/specs/page/page-status/index.js

@@ -6,7 +6,6 @@ import { errorWithMessageOnFail } from '../../../../utils';
 export default async ( url, status ) => {
 	return errorWithMessageOnFail(
 		`Expected to received a 200 status for ${ url }. Received ${ status }.`,
-		'page-should-return-200-status',
 		() => {
 			expect( status ).toBe( 200 );
 		}

actions/ui-check/tests/e2e/specs/page/php-errors/index.js

@@ -11,8 +11,9 @@ export default async ( url ) => {
 	const pageError = await getPageError();
 
 	return errorWithMessageOnFail(
-		`${ url } contains PHP errors: ${ removeLocalPathRefs( pageError ) }`,
-		'page-should-not-have-php-errors',
+		`Loading the page using ${ url } contains PHP errors: ${ removeLocalPathRefs(
+			pageError
+		) }`,
 		() => {
 			expect( pageError ).toBe( null );
 		}

actions/ui-check/tests/e2e/specs/page/unexpected-links/index.js

@@ -62,7 +62,6 @@ export default async ( url ) => {
 		let hostname = removeWWW( href_url.hostname );
 		const result = errorWithMessageOnFail(
 			`"${ hostname }" found when viewing ${ url } is not an approved link.`,
-			'page-should-not-have-unexpected-links',
 			() => {
 				expect( allowed_hosts ).toContain( hostname );
 			}

actions/ui-check/tests/e2e/specs/ui/element-focus/index.js

@@ -124,13 +124,9 @@ export default async () => {
 		return await test();
 	} catch ( ex ) {
 		if ( ex instanceof FailedTestException ) {
-			warnWithMessageOnFail(
-				ex.messages,
-				'should-have-element-focus-state',
-				() => {
-					expect( false ).toEqual( true );
-				}
-			);
+			warnWithMessageOnFail( ex.messages, () => {
+				expect( false ).toEqual( true );
+			} );
 		} else {
 			console.log( ex );
 		}

actions/ui-check/tests/e2e/specs/ui/index.test.js

@@ -8,7 +8,7 @@ import subMenuTest from './sub-menu';
 import elementFocusTest from './element-focus';
 import tabbingTest from './tabbing';
 
-describe( 'Accessibility: UI', () => {
+describe( '/', () => {
 	beforeAll( async () => {
 		await goTo( '/' );
 	} );

actions/ui-check/tests/e2e/specs/ui/skip-links/index.js

@@ -56,13 +56,9 @@ export default async () => {
 		return await test();
 	} catch ( ex ) {
 		if ( ex instanceof FailedTestException ) {
-			return warnWithMessageOnFail(
-				ex.messages,
-				'should-have-skip-links',
-				() => {
-					expect( false ).toEqual( true );
-				}
-			);
+			return warnWithMessageOnFail( ex.messages, () => {
+				expect( false ).toEqual( true );
+			} );
 		} else {
 			console.log( ex );
 		}

actions/ui-check/tests/e2e/specs/ui/sub-menu/index.js

@@ -100,16 +100,10 @@ export default async () => {
 		return await test();
 	} catch ( ex ) {
 		if ( ex instanceof FailedTestException ) {
-			warnWithMessageOnFail(
-				ex.messages,
-				'should-have-appropriate-submenus',
-
-				() => {
-					expect( false ).toEqual( true );
-				}
-			);
+			warnWithMessageOnFail( ex.messages, () => {
+				expect( false ).toEqual( true );
+			} );
 		} else {
-			console.log( ex );
 		}
 
 		return false;

actions/ui-check/tests/e2e/specs/ui/tabbing/index.js

@@ -92,14 +92,9 @@ export default async () => {
 				await makeGif( 1280, 800, SCREENSHOT_TABBING_TEST, 100 );
 			}
 
-			warnWithMessageOnFail(
-				ex.messages,
-				'should-have-logical-tabbing',
-
-				() => {
-					expect( false ).toEqual( true );
-				}
-			);
+			warnWithMessageOnFail( ex.messages, () => {
+				expect( false ).toEqual( true );
+			} );
 		} else {
 			console.log( ex );
 		}