Speed up testing, docs. Start exporting fixtures. (#84)

* update pytest to introduce smoke testing.
* rname selectors cause of subprocess conflict
* make a reusable axe pytest fixture
* get axe from npm in ci
* fluent api for axe results object in fixture
* drop convert script and build docs
* remove old docs build and prefer the new system
* name the test workflow
* export utility functiuons for axe test
* add tests for a11y dialogs
* rname test_playwirght

.github/workflows/test.yml

@@ -1,5 +1,6 @@
+name: pytest nbconvert-a11y, axe test exports, build docs.
 on:
-- push
+  - push
 jobs:
   pypi:
     defaults:
@@ -8,33 +9,52 @@ jobs:
     strategy:
       matrix:
         python-version:
-        - "3.10"
+          - "3.10"
     runs-on: ubuntu-latest
     steps:
-    - name: fetch all history and tags
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Cache conda
-      uses: actions/cache@v2
-      env:
-        # Increase this value to reset cache if etc/example-environment.yml has not changed
-        CACHE_NUMBER: 0
-      with:
-        path: ~/conda_pkgs_dir
-        key:
-          ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
-          hashFiles('test-environment.yml') }}
-    - uses: mamba-org/setup-micromamba@v1
-      with:
-        environment-file: test-environment.yml
-        cache-environment: true
-    - name: init playwright nbconvert-a11y
-      run: |
-        playwright install --with-deps chromium
-        npm install vnu-jar
-        pip install -e.
-        doit copy
-    - name: test with pytest
-      run: |
-        pytest
+      - name: fetch all history and tags
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Cache conda
+        uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
+            hashFiles('test-environment.yml') }}
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: test-environment.yml
+          cache-environment: true
+      - name: init playwright nbconvert-a11y
+        run: |
+          playwright install --with-deps chromium
+          npm install vnu-jar axe-core
+          pip install -e.
+          doit copy
+      - name: test with pytest
+        run: |
+          # the smoke generate html assets that are used in the accessibility testing.
+          # we run this script to generate assets and test the nbconvert-a11y module.
+          pytest tests/test_smoke.py
+          pytest --deselect tests/test_smoke.py
+      - name: mkdocs
+        run: |
+          mkdocs build -v
+      - name: Deploy main 🚀
+        uses: JamesIves/github-pages-deploy-action@v4
+        if: ${{ github.ref_name == 'main' }}
+        with:
+          folder: site # The folder the action should deploy.
+          single-commit: true
+      - name: Deploy non-main 🚀
+        uses: JamesIves/github-pages-deploy-action@v4
+        if: ${{ github.ref_name != 'main' }}
+        with:
+          folder: site # The folder the action should deploy.
+          single-commit: true
+          target-folder: branch/${{ github.ref_name }}                  
+

nbconvert_html5/exporters.py

@@ -8,7 +8,7 @@
 from nbconvert.exporters.html import HTMLExporter
 from nbconvert.preprocessors import CSSHTMLHeaderPreprocessor
 from bs4 import BeautifulSoup, Tag
-from .selectors import MAIN, CODE, MD, OUT, PROMPT
+from ._selectors import MAIN, CODE, MD, OUT, PROMPT
 from re import compile
 
 DIR = Path(__file__).parent

nbconvert_html5/pytest_axe.py

@@ -0,0 +1,131 @@
+# requires node and axe
+# requires playwright
+import dataclasses
+from functools import lru_cache
+from itertools import chain
+from json import dumps, loads
+from os import environ
+from shlex import quote, split
+from subprocess import CalledProcessError, check_output
+from sys import argv
+from pathlib import Path
+from typing import Any
+from attr import dataclass
+import exceptiongroup
+from pytest import fixture, mark, param
+import pytest
+
+NBCONVERT_HTML5_DYNAMIC_TEST = "NBCONVERT_HTML5_DYNAMIC_TEST"
+
+axe_config_aa = {
+    "runOnly": ["act", "best-practice", "experimental", "wcag21a", "wcag21aa", "wcag22aa"],
+    "allowedOrigins": ["<same_origin>"],
+}
+
+axe_config_aaa = {
+    "runOnly": [
+        "act",
+        "best-practice",
+        "experimental",
+        "wcag21a",
+        "wcag21aa",
+        "wcag22aa",
+        "wcag2aaa",
+    ],
+    "allowedOrigins": ["<same_origin>"],
+}
+
+MATHJAX = "[id^=MathJax]"
+tests_axe = {"exclude": [MATHJAX]}
+
+
+def get_npm_directory(package, data=False):
+    try:
+        info = loads(check_output(split(f"npm ls --long --depth 0 --json {quote(package)}")))
+    except CalledProcessError:
+        return
+    if data:
+        return info
+    return Path(info.get("dependencies").get(package).get("path"))
+
+
+@dataclass
+class AxeResults:
+    data: Any
+
+    def raises(self):
+        if self.data["violations"]:
+            raise AxeException.from_violations(self.data)
+        return self
+
+    def dump(self, file: Path):
+        if file.is_dir():
+            file /= "axe-results.json"
+        file.parent.mkdir(exist_ok=True, parents=True)
+        file.write_text(dumps(self.data))
+        return self
+
+
+@dataclasses.dataclass
+class AxeException(Exception):
+    message: str
+    target: list
+    data: dict = dataclasses.field(repr=False)
+
+    types = {}
+
+    @classmethod
+    def new(cls, id, impact, message, data, target, **kwargs):
+        if id in cls.types:
+            cls = cls.types.get(id)
+        else:
+            cls = cls.types.setdefault(
+                id,
+                type(
+                    f"{impact.capitalize()}{''.join(map(str.capitalize, id.split('-')))}Exception",
+                    (cls,),
+                    dict(),
+                ),
+            )
+        return cls(message, target, data)
+
+    @classmethod
+    def from_violations(cls, data):
+        out = []
+        for violation in (violations := data.get("violations")):
+            for node in violation["nodes"]:
+                for exc in node["any"]:
+                    out.append(cls.new(**exc, target=["target"]))
+        return exceptiongroup.ExceptionGroup(f"{len(violations)} accessibility violations", out)
+
+
+@mark.parametrize("package", ["axe-core", param("axe-core-doesnt-ship-this", marks=mark.xfail)])
+def test_non_package(package):
+    assert get_npm_directory(package), "package not found."
+
+
+@lru_cache(1)
+def get_axe():
+    return (get_npm_directory("axe-core") / "axe.js").read_text()
+
+
+def inject_axe(page):
+    page.evaluate(get_axe())
+
+
+def run_axe_test(page, tests_config=None, axe_config=None):
+    return AxeResults(
+        page.evaluate(
+            f"window.axe.run({tests_config and dumps(tests_config) or 'document'}, {dumps(axe_config or {})})"
+        )
+    )
+
+
+@fixture
+def axe(page):
+    def go(url, tests=tests_axe, axe_config=axe_config_aa):
+        page.goto(url)
+        inject_axe(page)
+        return run_axe_test(page, tests, axe_config)
+
+    yield go

pyproject.toml

@@ -66,6 +66,10 @@ serve = "mkdocs serve -v"
 html5_test = "nbconvert_html5.exporters:Html5Test"
 a11y = "nbconvert_html5.form_exporter:A11yExporter"
 
+[project.entry-points.pytest11]
+axe = "nbconvert_html5.pytest_axe"
+a11y = "nbconvert_html5.form_exporter:A11yExporter"
+
 [tool.hatch.build.targets.wheel.shared-data]
 "nbconvert_html5/templates" = "share/jupyter/nbconvert/templates"
 
@@ -85,7 +89,7 @@ run = "pytest"
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "--nbval --nbval-current-env -pno:importnb -n auto --nbval-sanitize-with=sanitize.cfg"
+addopts = "-vvv --nbval --nbval-current-env -pno:importnb -n auto --nbval-sanitize-with=sanitize.cfg"
 testpaths = ["tests", "test_playwright.py"]
 norecursedirs = ["tests/exports", "tests/notebooks", "*checkpoints"]
 

test-environment.yml

@@ -26,3 +26,5 @@ dependencies:
   - beautifulsoup4
   - scipy
   - doit
+  - mkdocs-material
+  - mkdocstrings[python]