From fd10da56e6a90fef98433e1a663372599b06ea00 Mon Sep 17 00:00:00 2001 From: Tony Fast Date: Sun, 19 Nov 2023 19:37:34 -0800 Subject: [PATCH] Speed up testing, docs. Start exporting fixtures. (#84) * update pytest to introduce smoke testing. * rname selectors cause of subprocess conflict * make a reusable axe pytest fixture * get axe from npm in ci * fluent api for axe results object in fixture * drop convert script and build docs * remove old docs build and prefer the new system * name the test workflow * export utility functiuons for axe test * add tests for a11y dialogs * rname test_playwirght --- .github/workflows/convert.yml | 53 ------- .github/workflows/docs.yml | 41 ------ .github/workflows/test.yml | 78 +++++++---- .../{selectors.py => _selectors.py} | 0 nbconvert_html5/exporters.py | 2 +- nbconvert_html5/pytest_axe.py | 131 ++++++++++++++++++ pyproject.toml | 6 +- test-environment.yml | 2 + tests/test_axe.py | 129 +++++++++++++++++ tests/test_playwright.py | 98 ------------- tests/test_smoke.py | 94 +++++++++++++ 11 files changed, 411 insertions(+), 223 deletions(-) delete mode 100644 .github/workflows/convert.yml delete mode 100644 .github/workflows/docs.yml rename nbconvert_html5/{selectors.py => _selectors.py} (100%) create mode 100644 nbconvert_html5/pytest_axe.py create mode 100644 tests/test_axe.py delete mode 100644 tests/test_playwright.py create mode 100644 tests/test_smoke.py diff --git a/.github/workflows/convert.yml b/.github/workflows/convert.yml deleted file mode 100644 index 69bb3242..00000000 --- a/.github/workflows/convert.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: build nbconvert_html5 and export samples - -on: - push: - branches-ignore: - - gh-pages - paths: - - .github/workflows/** - - nbconvert_html5/** - - "pyproject.toml" - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - - steps: - - uses: actions/checkout@v3 - name: Fetch all history for all tags and branches - with: - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: | - pyproject.toml - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install hatch - - name: Convert - run: | - python -m hatch run task:convert - - name: Build the docs - run: | - python -m hatch run docs:build - - name: Deploy main 🚀 - uses: JamesIves/github-pages-deploy-action@v4 - if: ${{ github.ref_name == 'main' }} - with: - folder: site # The folder the action should deploy. - single-commit: true - - name: Deploy non-main 🚀 - uses: JamesIves/github-pages-deploy-action@v4 - if: ${{ github.ref_name != 'main' }} - with: - folder: site # The folder the action should deploy. - single-commit: true - target-folder: branch/${{ github.ref_name }} \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index fdf5dc40..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: build documentation - -on: [push] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10"] - - steps: - - uses: actions/checkout@v3 - name: Fetch all history for all tags and branches - with: - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: '**/pyproject.toml' - - name: Install dependencies - run: python -m pip install --upgrade pip hatch - - name: Build mkdocs - run: | - python -m hatch run task:export - python -m hatch run docs:build - - name: Deploy main 🚀 - uses: JamesIves/github-pages-deploy-action@v4 - if: ${{ github.ref_name == 'main' }} - with: - folder: site # The folder the action should deploy. - single-commit: true - - name: Deploy non-main 🚀 - uses: JamesIves/github-pages-deploy-action@v4 - if: ${{ github.ref_name != 'main' }} - with: - folder: site # The folder the action should deploy. - single-commit: true - target-folder: branch/${{ github.ref_name }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d410cca5..50e931d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,6 @@ +name: pytest nbconvert-a11y, axe test exports, build docs. on: -- push + - push jobs: pypi: defaults: @@ -8,33 +9,52 @@ jobs: strategy: matrix: python-version: - - "3.10" + - "3.10" runs-on: ubuntu-latest steps: - - name: fetch all history and tags - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - env: - # Increase this value to reset cache if etc/example-environment.yml has not changed - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('test-environment.yml') }} - - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: test-environment.yml - cache-environment: true - - name: init playwright nbconvert-a11y - run: | - playwright install --with-deps chromium - npm install vnu-jar - pip install -e. - doit copy - - name: test with pytest - run: | - pytest \ No newline at end of file + - name: fetch all history and tags + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Cache conda + uses: actions/cache@v2 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('test-environment.yml') }} + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: test-environment.yml + cache-environment: true + - name: init playwright nbconvert-a11y + run: | + playwright install --with-deps chromium + npm install vnu-jar axe-core + pip install -e. + doit copy + - name: test with pytest + run: | + # the smoke generate html assets that are used in the accessibility testing. + # we run this script to generate assets and test the nbconvert-a11y module. + pytest tests/test_smoke.py + pytest --deselect tests/test_smoke.py + - name: mkdocs + run: | + mkdocs build -v + - name: Deploy main 🚀 + uses: JamesIves/github-pages-deploy-action@v4 + if: ${{ github.ref_name == 'main' }} + with: + folder: site # The folder the action should deploy. + single-commit: true + - name: Deploy non-main 🚀 + uses: JamesIves/github-pages-deploy-action@v4 + if: ${{ github.ref_name != 'main' }} + with: + folder: site # The folder the action should deploy. + single-commit: true + target-folder: branch/${{ github.ref_name }} + \ No newline at end of file diff --git a/nbconvert_html5/selectors.py b/nbconvert_html5/_selectors.py similarity index 100% rename from nbconvert_html5/selectors.py rename to nbconvert_html5/_selectors.py diff --git a/nbconvert_html5/exporters.py b/nbconvert_html5/exporters.py index 71ea080b..2acae722 100644 --- a/nbconvert_html5/exporters.py +++ b/nbconvert_html5/exporters.py @@ -8,7 +8,7 @@ from nbconvert.exporters.html import HTMLExporter from nbconvert.preprocessors import CSSHTMLHeaderPreprocessor from bs4 import BeautifulSoup, Tag -from .selectors import MAIN, CODE, MD, OUT, PROMPT +from ._selectors import MAIN, CODE, MD, OUT, PROMPT from re import compile DIR = Path(__file__).parent diff --git a/nbconvert_html5/pytest_axe.py b/nbconvert_html5/pytest_axe.py new file mode 100644 index 00000000..fe96e5fb --- /dev/null +++ b/nbconvert_html5/pytest_axe.py @@ -0,0 +1,131 @@ +# requires node and axe +# requires playwright +import dataclasses +from functools import lru_cache +from itertools import chain +from json import dumps, loads +from os import environ +from shlex import quote, split +from subprocess import CalledProcessError, check_output +from sys import argv +from pathlib import Path +from typing import Any +from attr import dataclass +import exceptiongroup +from pytest import fixture, mark, param +import pytest + +NBCONVERT_HTML5_DYNAMIC_TEST = "NBCONVERT_HTML5_DYNAMIC_TEST" + +axe_config_aa = { + "runOnly": ["act", "best-practice", "experimental", "wcag21a", "wcag21aa", "wcag22aa"], + "allowedOrigins": [""], +} + +axe_config_aaa = { + "runOnly": [ + "act", + "best-practice", + "experimental", + "wcag21a", + "wcag21aa", + "wcag22aa", + "wcag2aaa", + ], + "allowedOrigins": [""], +} + +MATHJAX = "[id^=MathJax]" +tests_axe = {"exclude": [MATHJAX]} + + +def get_npm_directory(package, data=False): + try: + info = loads(check_output(split(f"npm ls --long --depth 0 --json {quote(package)}"))) + except CalledProcessError: + return + if data: + return info + return Path(info.get("dependencies").get(package).get("path")) + + +@dataclass +class AxeResults: + data: Any + + def raises(self): + if self.data["violations"]: + raise AxeException.from_violations(self.data) + return self + + def dump(self, file: Path): + if file.is_dir(): + file /= "axe-results.json" + file.parent.mkdir(exist_ok=True, parents=True) + file.write_text(dumps(self.data)) + return self + + +@dataclasses.dataclass +class AxeException(Exception): + message: str + target: list + data: dict = dataclasses.field(repr=False) + + types = {} + + @classmethod + def new(cls, id, impact, message, data, target, **kwargs): + if id in cls.types: + cls = cls.types.get(id) + else: + cls = cls.types.setdefault( + id, + type( + f"{impact.capitalize()}{''.join(map(str.capitalize, id.split('-')))}Exception", + (cls,), + dict(), + ), + ) + return cls(message, target, data) + + @classmethod + def from_violations(cls, data): + out = [] + for violation in (violations := data.get("violations")): + for node in violation["nodes"]: + for exc in node["any"]: + out.append(cls.new(**exc, target=["target"])) + return exceptiongroup.ExceptionGroup(f"{len(violations)} accessibility violations", out) + + +@mark.parametrize("package", ["axe-core", param("axe-core-doesnt-ship-this", marks=mark.xfail)]) +def test_non_package(package): + assert get_npm_directory(package), "package not found." + + +@lru_cache(1) +def get_axe(): + return (get_npm_directory("axe-core") / "axe.js").read_text() + + +def inject_axe(page): + page.evaluate(get_axe()) + + +def run_axe_test(page, tests_config=None, axe_config=None): + return AxeResults( + page.evaluate( + f"window.axe.run({tests_config and dumps(tests_config) or 'document'}, {dumps(axe_config or {})})" + ) + ) + + +@fixture +def axe(page): + def go(url, tests=tests_axe, axe_config=axe_config_aa): + page.goto(url) + inject_axe(page) + return run_axe_test(page, tests, axe_config) + + yield go diff --git a/pyproject.toml b/pyproject.toml index 0d2ce59c..94469e33 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,10 @@ serve = "mkdocs serve -v" html5_test = "nbconvert_html5.exporters:Html5Test" a11y = "nbconvert_html5.form_exporter:A11yExporter" +[project.entry-points.pytest11] +axe = "nbconvert_html5.pytest_axe" +a11y = "nbconvert_html5.form_exporter:A11yExporter" + [tool.hatch.build.targets.wheel.shared-data] "nbconvert_html5/templates" = "share/jupyter/nbconvert/templates" @@ -85,7 +89,7 @@ run = "pytest" [tool.pytest.ini_options] minversion = "6.0" -addopts = "--nbval --nbval-current-env -pno:importnb -n auto --nbval-sanitize-with=sanitize.cfg" +addopts = "-vvv --nbval --nbval-current-env -pno:importnb -n auto --nbval-sanitize-with=sanitize.cfg" testpaths = ["tests", "test_playwright.py"] norecursedirs = ["tests/exports", "tests/notebooks", "*checkpoints"] diff --git a/test-environment.yml b/test-environment.yml index 3ef48c81..ea3f33bb 100644 --- a/test-environment.yml +++ b/test-environment.yml @@ -26,3 +26,5 @@ dependencies: - beautifulsoup4 - scipy - doit + - mkdocs-material + - mkdocstrings[python] \ No newline at end of file diff --git a/tests/test_axe.py b/tests/test_axe.py new file mode 100644 index 00000000..078fe192 --- /dev/null +++ b/tests/test_axe.py @@ -0,0 +1,129 @@ +import contextlib +import dataclasses +from json import dumps, loads +from logging import getLogger +from pathlib import Path + +import exceptiongroup +from nbconvert_html5.pytest_axe import inject_axe, run_axe_test +from test_nbconvert_html5 import exporter + + +from pytest import fixture, mark, param, xfail + +from tests.test_smoke import CONFIGURATIONS, NOTEBOOKS, SKIPCI, get_target_html + +TPL_NOT_ACCESSIBLE = mark.xfail(reason="template is not accessible") +HERE = Path(__file__).parent +EXPORTS = HERE / "exports" +HTML = EXPORTS / "html" +LOGGER = getLogger(__name__) +AUDIT = EXPORTS / "audit" + +# ignore mathjax at the moment. we might be able to turne mathjax to have better +# accessibility. https://github.com/Iota-School/notebooks-for-all/issues/81 +MATHJAX = "[id^=MathJax]" +NEEDS_WORK = "state needs work" + + +aa_config_notebooks = mark.parametrize( + "config,notebook", + [ + param( + (CONFIGURATIONS / (a := "a11y")).with_suffix(".py"), + (NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"), + id="-".join((b, a)), + ), + param( + (CONFIGURATIONS / (a := "default")).with_suffix(".py"), + (NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"), + marks=[SKIPCI, TPL_NOT_ACCESSIBLE], + id="-".join((b, a)), + ), + ], +) + +aaa_config_notebooks = mark.parametrize( + "config,notebook", + [ + param( + (CONFIGURATIONS / (a := "a11y")).with_suffix(".py"), + (NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"), + id="-".join( + (b, a), + ), + marks=[TPL_NOT_ACCESSIBLE], + ) + ], +) + + +axe_config_aaa = { + "runOnly": [ + "act", + "best-practice", + "experimental", + "wcag21a", + "wcag21aa", + "wcag22aa", + "wcag2aaa", + ], + "allowedOrigins": [""], +} + + +@aa_config_notebooks +def test_axe_aa(axe, config, notebook): + target = get_target_html(config, notebook) + audit = AUDIT / target.with_suffix(".json").name + axe(Path.as_uri(target)).dump(audit).raises() + + +@aaa_config_notebooks +def test_axe_aaa(axe, config, notebook): + target = get_target_html(config, notebook) + audit = AUDIT / target.with_suffix(".json").name + axe(Path.as_uri(target), axe_config=axe_config_aaa).dump(audit).raises() + + +config_notebooks_dialog = mark.parametrize( + "config,notebook", + [ + param( + (CONFIGURATIONS / (a := "a11y")).with_suffix(".py"), + (NOTEBOOKS / (b := "lorenz-executed")).with_suffix(".ipynb"), + id="-".join( + (b, a), + ), + ) + ], +) + + +@fixture(scope="function") +def axe_page(page): + def go(url): + page.goto(url) + inject_axe(page) + return page + + return go + + +@config_notebooks_dialog +@mark.parametrize( + "dialog", + [ + param("[aria-controls=nb-settings]", marks=mark.xfail(reason=NEEDS_WORK)), + "[aria-controls=nb-help]", + "[aria-controls=nb-metadata]", + "[aria-controls=nb-audit]", + param("[aria-controls=nb-expanded-dialog]", marks=mark.xfail(reason=NEEDS_WORK)), + param("[aria-controls=nb-visibility-dialog]", marks=mark.xfail(reason=NEEDS_WORK)), + param("nada", marks=mark.xfail(reason="no selector")), + ], +) +def test_dialogs(axe_page, config, notebook, dialog): + page = axe_page(get_target_html(config, notebook).absolute().as_uri()) + page.click(dialog) + run_axe_test(page).raises() diff --git a/tests/test_playwright.py b/tests/test_playwright.py deleted file mode 100644 index 5b822674..00000000 --- a/tests/test_playwright.py +++ /dev/null @@ -1,98 +0,0 @@ -import dataclasses -from json import dumps, loads -from logging import getLogger -from pathlib import Path - -import exceptiongroup -from test_nbconvert_html5 import exporter - - -from pytest import fixture, mark - -HERE = Path(__file__).parent -NOTEBOOKS = HERE / "notebooks" -EXPORTS = HERE / "exports" -HTML = EXPORTS / "html" -LOGGER = getLogger(__name__) -AUDIT = EXPORTS / "audit" - -# ignore mathjax at the moment. we might be able to turne mathjax to have better -# accessibility. https://github.com/Iota-School/notebooks-for-all/issues/81 -MATHJAX = "[id^=MathJax]" - - -@dataclasses.dataclass -class AxeException(Exception): - message: str - target: list - data: dict = dataclasses.field(repr=False) - - types = {} - - @classmethod - def new(cls, id, impact, message, data, target, **kwargs): - if id in cls.types: - cls = cls.types.get(id) - else: - cls = cls.types.setdefault( - id, - type( - f"{impact.capitalize()}{''.join(map(str.capitalize, id.split('-')))}Exception", - (cls,), - dict(), - ), - ) - return cls(message, target, data) - - @classmethod - def from_violations(cls, data): - out = [] - for violation in (violations := data.get("violations")): - for node in violation["nodes"]: - for exc in node["any"]: - out.append(cls.new(**exc, target=["target"])) - return exceptiongroup.ExceptionGroup(f"{len(violations)} accessibility violations", out) - - -@fixture -def axe(): - from requests_cache import install_cache - import requests - - install_cache("a11y-audit") - yield requests.get("https://cdn.jsdelivr.net/npm/axe-core").text - - -@mark.parametrize( - "notebook", - list( - x - for x in NOTEBOOKS.glob("*.ipynb") - if x.name not in {"Imaging_Sky_Background_Estimation.ipynb"} - ), -) -def test_baseline_a11y_template(page, exporter, notebook, axe): - config = {} - config.setdefault( - "runOnly", - ["act", "best-practice", "experimental", "wcag21a", "wcag21aa", "wcag22aa"], - ) - config.setdefault("allowedOrigins", [""]) - target = HTML / notebook.with_suffix(".html").name - target.parent.mkdir(exist_ok=True, parents=True) - LOGGER.debug(f"""injecting axe into {target.name}""") - target.write_text(exporter.from_filename(notebook)[0]) - - test = dict(exclude=[MATHJAX]) - page.goto(Path.as_uri(target)) - LOGGER.debug(f"""injecting axe into {target.name}""") - page.evaluate(axe) - LOGGER.debug(f"""auditting {target.name} with axe""") - result = page.evaluate(f"window.axe.run({dumps(test)}, {dumps(config)})") - AUDIT.mkdir(parents=True, exist_ok=True) - audit = AUDIT / notebook.with_suffix(".json").name - LOGGER.info(f"""writing {audit} with {len(result["violations"])} violations""") - audit.write_text(dumps(result)) - - if result["violations"]: - raise AxeException.from_violations(result) diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 00000000..b8bfd06c --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,94 @@ +"""smoke tests that verify the python module execution. +these tests do not invoke any browsers. + +the tests verify: +* the nbconvert exporters are properly exported. +* config files are applied to exports +""" + +from functools import lru_cache +import itertools +from logging import getLogger +from os import environ +from pathlib import Path +from shutil import copyfile +from pytest import fixture, mark, param +import nbconvert.nbconvertapp +import nbconvert_html5 + +TEMPLATES = Path(nbconvert_html5.__file__).parent / "templates/a11y" +SKIP_BASELINE = "baseline tests skipped locally" +LOGGER = getLogger(__name__) +HERE = Path(__file__).parent +CONFIGURATIONS = HERE / "configurations" +NOTEBOOKS = HERE / "notebooks" +EXPORTS = HERE / "exports" +HTML = EXPORTS / "html" +CI = environ.get("CI") +SKIPCI = mark.skipif(not CI, reason=SKIP_BASELINE) + + +@lru_cache +def exporter_from_config(config: Path) -> nbconvert.Exporter: + """create a nbconvert exporter from an IPython configuration file.""" + app = nbconvert.nbconvertapp.NbConvertApp(config_file=str(config)) + app.load_config_file() + exporter = nbconvert.get_exporter(app.export_format)(config=app.config) + return exporter + + +def get_target_html(config, notebook): + target = (HTML / "-".join([notebook.stem, config.stem])).with_suffix(".html") + target.parent.mkdir(exist_ok=True, parents=True) + return target + + +configs = mark.parametrize( + "config", + [ + param((CONFIGURATIONS / (id := "a11y")).with_suffix(".py"), id=id), + param( + (CONFIGURATIONS / (id := "default")).with_suffix(".py"), + id=id, + marks=SKIPCI, + ), + ], +) + +notebooks = mark.parametrize( + "notebook", + [ + param((NOTEBOOKS / (id := "lorenz")).with_suffix(".ipynb"), id=id), + param((NOTEBOOKS / (id := "lorenz-executed")).with_suffix(".ipynb"), id=id), + ], +) + + +assets = mark.parametrize("assets", [TEMPLATES / "settings.js", TEMPLATES / "style.css"]) + + +@configs +def test_config_loading(config): + """verify configs are loaded.""" + exporter_from_config(config) # will ExporterNameError if there is a failure. + + +@assets +def test_static_assets(assets): + """this is a bad test. it won't fail, but needs to run to collect testing assets.""" + target = HTML / assets.name + try: + assert target.exists(), f"{assets.name} doesn't exist." + except AssertionError: + copyfile(assets, target) + assert target.exists(), f"{assets.name} couldn't be created" + + +@configs +@notebooks +def test_export_notebooks(config, notebook): + """verify that all the internals work sufficiently to export notebooks.""" + html, resources = exporter_from_config(config).from_filename(notebook) + TARGET = get_target_html(config, notebook) + TARGET.write_text(html) + LOGGER.debug(f"writing html to {TARGET}")