Add e2e job for accessibility tests

.github/workflows/test.yaml

@@ -140,7 +140,66 @@ jobs:
           name: ${{github.run_number}}-${{github.run_attempt}}-screenshots-${{ matrix.role.tag }}+${{ matrix.features[0] }}
           path: cypress/screenshots
 
+  a11y-test:
+    if: "!contains( github.event.pull_request.labels.*.name, 'ci/skip-e2e')"
+    needs: e2e-ui-build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Setup env
+        uses: ./.github/actions/setup
 
+      # Installing fixed version of Chrome since latest version does not work (128 didn't work)
+      # Leaving this here again in case we need to pin to a specific Chrome version in the future
+      - name: Install Chrome 127
+        run: |
+          sudo apt-get install -y wget libu2f-udev
+          cd /tmp
+          wget -q http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_127.0.6533.72-1_amd64.deb
+          sudo dpkg -i google-chrome-stable_127.0.6533.72-1_amd64.deb
+          sudo apt-get install -y --allow-downgrades ./google-chrome-stable_127.0.6533.72-1_amd64.deb
+          google-chrome --version
+
+      - name: Download e2e build
+        uses:  actions/download-artifact@v4
+        with:
+          name: ${{ env.E2E_BUILD_DIST_NAME }}
+          path: ${{ env.E2E_BUILD_DIST_DIR }}
+      - name: Download e2e build ember
+        uses:  actions/download-artifact@v4
+        with:
+          name: ${{ env.E2E_BUILD_DIST_EMBER_NAME }}
+          path: ${{ env.E2E_BUILD_DIST_EMBER_DIR }}
+
+      - name: Run Rancher
+        run: yarn e2e:docker
+
+      - name: Setup Rancher and user
+        run: |
+          yarn e2e:prod
+        env: 
+          GREP_TAGS: "@adminSetup+@accessibility --@jenkins"
+          TEST_USERNAME: admin
+          TEST_ONLY: setup
+      - name: Run user tests
+        run: |
+          yarn e2e:prod
+          [ "$BUILD_DASHBOARD" != "false" ] || exit 0
+        env: 
+          TEST_A11Y: true
+          TEST_SKIP: setup
+          GREP_TAGS: "@admin+@accessibility --@jenkins"
+          TEST_USERNAME: admin
+
+      # - name: Upload screenshots
+      #   uses: actions/upload-artifact@v4
+      #   if: ${{ failure() }}
+      #   with:
+      #     name: ${{github.run_number}}-${{github.run_attempt}}-screenshots-${{ matrix.role.tag }}+${{ matrix.features[0] }}
+      #     path: cypress/screenshots
+
   unit-test:
     runs-on: ubuntu-latest
     steps:

cypress.config.ts

@@ -11,17 +11,15 @@ require('dotenv').config();
  * VARIABLES
  */
 const hasCoverage = (process.env.TEST_INSTRUMENT === 'true') || false; // Add coverage if instrumented
-let testDirs = ['accessibility', 'priority', 'components', 'setup', 'pages', 'navigation', 'global-ui', 'features', 'extensions'];
+let testDirs = ['priority', 'components', 'setup', 'pages', 'navigation', 'global-ui', 'features', 'extensions'];
 const skipSetup = process.env.TEST_SKIP?.includes('setup');
 const baseUrl = (process.env.TEST_BASE_URL || 'https://localhost:8005').replace(/\/$/, '');
 const DEFAULT_USERNAME = 'admin';
 const username = process.env.TEST_USERNAME || DEFAULT_USERNAME;
 const apiUrl = process.env.API || (baseUrl.endsWith('/dashboard') ? baseUrl.split('/').slice(0, -1).join('/') : baseUrl);
 
-if (process.env.TEST_DIRS) {
-  testDirs = process.env.TEST_DIRS.split(',').map((s) => s.trim());
-
-  console.log(` Using test dirs: ${ testDirs }`);
+if (process.env.TEST_A11Y) {
+  testDirs = ['accessibility'];
 }
 
 /**

cypress/e2e/tests/accessibility/login.spec.ts

@@ -1,19 +1,23 @@
-// const WAIT = 600;
+import { LoginPagePo } from '@/cypress/e2e/po/pages/login-page.po';
+
+describe('Login page a11y testing', { tags: ['@adminUser', '@accessibility'] }, () => {
+  const loginPage = new LoginPagePo();
 
-describe('Login page a11y testing', { tags: ['@adminUser', '@standardUser'] }, () => {
   it('wcag21aa test', () => {
-    cy.visit(`${ Cypress.config().baseUrl }/auth/login?local`);
+    loginPage.goTo();
+    loginPage.waitForPage();
+
     cy.injectAxe();
-    // cy.wait(WAIT);
-    cy.checkAccessibility();
+    cy.checkPageAccessibility();
   });
 
-  it('username fieldt', () => {
-    cy.visit(`${ Cypress.config().baseUrl }/auth/login?local`);
-    cy.injectAxe();
-    // cy.wait(WAIT);
-    // cy.checkPageAccessibility();
+  it('locale selector', () => {
+    loginPage.goTo();
+    loginPage.waitForPage();
 
-    cy.checkAccessibility('#username');
+    cy.injectAxe();
+    cy.get('[data-testid="locale-selector"]').click();
+    cy.checkPageAccessibility();
+    cy.checkElementAccessibility('#username', 'Username field checks');
   });
 });

cypress/globals.d.ts

@@ -166,7 +166,12 @@ declare global {
       /**
        * Run an accessibility check on the current page or the specified element
        */
-      checkAccessibility(selector?: string);
+      checkPageAccessibility(description?: string);
+
+      /**
+       * Run an accessibility check on the specified element
+       */
+      checkElementAccessibility(selector: any, description?: string);
     }
   }
 }

cypress/support/commands/accessiblity.ts

@@ -1,16 +1,22 @@
-import { createHtmlReport } from 'axe-html-reporter';
+import { a11yScreenshot } from '../plugins/accessibility';
 
 // Custom violation callback function that prints a list of violations
+// Used when logging to the Cypress log
 const severityIndicators = {
   minor:    '⚪',
   moderate: '🟡',
   serious:  '🟠',
   critical: '🔴',
 };
 
+// Ignore color contrast for now
 const RULES = { rules: { 'color-contrast': { enabled: false } } };
 
-// Define at the top of the spec file or just import it
+// Used to track where multiple checks are done in a test to ensure we save
+// the screenshots for them to unique filenames
+const screenshotIndexes: {[key: string]: number} = {};
+
+// Log violations to the terminal
 function terminalLog(violations) {
   cy.task(
     'log',
@@ -33,50 +39,116 @@ function terminalLog(violations) {
   cy.task('table', violationData);
 }
 
-function logToFile(violations) {
-  cy.writeFile('accessibilityReport.json', `${ JSON.stringify(violations, null, 2) } \n`, { flag: 'a+' });
-}
-
-function printAccessibilityViolations(violations) {
-  // Log to the console
-  terminalLog(violations);
-  logToFile(violations);
+/**
+ * Log the violations in several ways:
+ * 1. Log to the terminal
+ * 2. Log to a file
+ * 3. Log to the Cypress log
+ * 4. Save screenshot of the violations
+ */
+function getAccessibilityViolationsCallback(description?: string) {
+  return function printAccessibilityViolations(violations) {
+    terminalLog(violations); // Log to the console
 
-  cy.task('a11y', violations);
+    const title = Cypress.currentTest.titlePath.join(', ');
+    const index = screenshotIndexes[title] || 1;
+    const testPath = Cypress.currentTest.titlePath;
+    const lastName = Cypress.currentTest.titlePath[Cypress.currentTest.titlePath.length - 1];
 
-  // Log in Cypress
-  violations.forEach((violation) => {
-    const nodes = Cypress.$(violation.nodes.map((item) => item.target).join(','));
+    testPath.push(description || `${ lastName } (#${ index })`);
 
-    Cypress.log({
-      name:         `${ severityIndicators[violation.impact] } A11y`,
-      consoleProps: () => violation,
-      $el:          nodes,
-      message:      `[${ violation.help }][${ violation.helpUrl }]`
+    cy.task('a11y', {
+      violations,
+      titlePath: testPath,
     });
 
-    violation.nodes.forEach(({ target }) => {
+    // Log in Cypress
+    violations.forEach((violation) => {
+      const nodes = Cypress.$(violation.nodes.map((item) => item.target).join(','));
+
       Cypress.log({
-        name:         `🔨`,
+        name:         `${ severityIndicators[violation.impact] } A11y`,
         consoleProps: () => violation,
-        $el:          Cypress.$(target.join(',')),
-        message:      target
+        $el:          nodes,
+        message:      `[${ violation.help }][${ violation.helpUrl }]`
       });
 
-      cy.get(target.join(', ')).then(($el) => {
-        $el.css('border', '2px solid red');
+      violation.nodes.forEach(({ target }) => {
+        Cypress.log({
+          name:         `🔨`,
+          consoleProps: () => violation,
+          $el:          Cypress.$(target.join(',')),
+          message:      target
+        });
+
+        // Store the existing border and change it to clearly show the elements with violations
+        cy.get(target.join(', ')).invoke('css', 'border').then((border) => {
+          cy.get(target.join(', ')).then(($el) => {
+            const existingBorder = $el.data('border');
+
+            // If we have the original border, don't store again = covers a case an element has multiple violations
+            // and we would lose the original border
+            if (!existingBorder) {
+              $el.data('border', border);
+            }
+
+            $el.css('border', '2px solid red');
+          });
+        });
       });
+    });
+
+    cy.screenshot(`a11y_${ Cypress.currentTest.title }_${ index }`);
 
-      // Highlight each node so that they are visible when we take a screenshot
-      // Cypress.$(target.join(',')).css('style', 'border: 2px solid red');
+    // cy.screenshot(`a11y_${ Cypress.currentTest.title }_${ index }`, {
+    //     onAfterScreenshot($el, props) {
+    //       a11yScreenshot({
+    //         titlePath: testPath,
+    //         props,
+    //       });
+    //   },
+    // });
+
+    cy.task('a11yScreenshot', {
+      titlePath: testPath,
+      name:      `a11y_${ Cypress.currentTest.title }_${ index }`
     });
-  });
 
-  cy.task('log', Cypress.currentTest);
-  cy.screenshot(`a11y_${ Cypress.currentTest.title }`);
+    screenshotIndexes[title] = index + 1;
+
+    // Reset the borders that were added to mark the elements with violations
+    violations.forEach((violation) => {
+      violation.nodes.forEach(({ target }) => {
+        cy.get(target.join(', ')).then(($el) => {
+          const border = $el.data('border');
+
+          if (!border.startsWith('0px none')) {
+            $el.css('border', $el.data('border'));
+          } else {
+            $el.css('border', '');
+          }
+
+          if ($el.attr('style')?.length === 0) {
+            $el.removeAttr('style');
+          }
+        });
+      });
+    });
+  }
 }
 
+/**
+ * Checks accessibility of the entire page
+ */
+// skipFailures = true will not fail the test when there are accessibility failures
+Cypress.Commands.add('checkPageAccessibility', (description?: string) => {
+  cy.checkA11y(undefined, RULES, getAccessibilityViolationsCallback(description), true);
+});
+
+/**
+ * Checks accessibility of a specific element
+ */
 // skipFailures = true will not fail the test when there are accessibility failures
-Cypress.Commands.add('checkAccessibility', (subject: any) => {
-  cy.checkA11y(subject, RULES, printAccessibilityViolations, true);
+Cypress.Commands.add('checkElementAccessibility', (subject: any, description?: string) => {
+  cy.checkA11y(subject, RULES, getAccessibilityViolationsCallback(description), true);
 });