From d00a094128c65b7bd27a6d8960cb8651b68c3106 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Wed, 10 May 2023 15:28:34 +0200 Subject: [PATCH] Test user workflow in dashboard (#581) * Add test for dashboard * Add playwright to dev dependencies * Improve test compatibility with chromium * Add github action for dashboard test * Move dashboard test to its own workflow * Do not run dashboard workflow test by default * Run dashboard tests when requested via --dashboard * Add some documentation --- .github/workflows/build.yml | 23 ++++ .github/workflows/sonarcloud.yml | 6 +- .pre-commit-config.yaml | 2 +- dianna/__init__.py | 1 - dianna/cli.py | 22 ++-- dianna/dashboard/Home.py | 1 - dianna/dashboard/_models_text.py | 1 - dianna/dashboard/_shared.py | 2 - dianna/dashboard/pages/1_Images.py | 1 - dianna/dashboard/pages/2_Text.py | 1 - dianna/dashboard/pages/3_Time_series.py | 1 - dianna/utils/tokenizers.py | 1 - docs/developer_info.rst | 26 +++++ pyproject.toml | 1 - setup.cfg | 6 +- setup.py | 1 - tests/conftest.py | 21 ++++ tests/methods/time_series_test_case.py | 1 - tests/test_dashboard.py | 146 ++++++++++++++++++++++++ tests/test_text_visualization.py | 1 - tests/utils.py | 1 - 21 files changed, 235 insertions(+), 31 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_dashboard.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 612d7a24..d918b596 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,11 +21,14 @@ jobs: runs-on: 'ubuntu-latest' steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/install-python-and-package with: python-version: '3.10' + - name: Run unit tests run: pytest -v + - name: Verify that we can build the package run: python3 setup.py sdist bdist_wheel @@ -40,10 +43,30 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/install-python-and-package with: python-version: ${{ matrix.python-version }} + - name: Run unit tests run: pytest -v + - name: Verify that we can build the package run: python3 setup.py sdist bdist_wheel + + test_dashboard: + name: Test dashboard + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-python-and-package + with: + python-version: '3.10' + extras-require: dev,dashboard + + - name: Ensure browser is installed + run: python -m playwright install chromium + + - name: Test dashboard + run: pytest -v -m dashboard --dashboard diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 10019413..cb236036 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -24,11 +24,15 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - uses: ./.github/actions/install-python-and-package + - name: Run unit tests with coverage - run: pytest --cov --cov-report term --cov-report xml --cov-report html --junitxml=xunit-result.xml tests/ + run: pytest --cov --cov-report term --cov-report xml --cov-report html --junitxml=xunit-result.xml tests/ -m "not dashboard" + - name: Correct coverage paths run: sed -i "s+$PWD/++g" coverage.xml + - name: SonarCloud Scan uses: SonarSource/sonarcloud-github-action@master env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48cad4d1..0bedf2b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: additional_dependencies: - toml - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.254' + rev: 'v0.0.264' hooks: - id: ruff args: [--fix] diff --git a/dianna/__init__.py b/dianna/__init__.py index 6cc2b792..70f0cdf3 100644 --- a/dianna/__init__.py +++ b/dianna/__init__.py @@ -24,7 +24,6 @@ import logging from . import utils - logging.getLogger(__name__).addHandler(logging.NullHandler()) __author__ = "DIANNA Team" diff --git a/dianna/cli.py b/dianna/cli.py index f805ec33..4c3e1977 100644 --- a/dianna/cli.py +++ b/dianna/cli.py @@ -1,6 +1,5 @@ import sys - if sys.version_info < (3, 10): from importlib_resources import files else: @@ -11,21 +10,18 @@ def dashboard(): """Start streamlit dashboard.""" from streamlit.web import cli as stcli - dashboard = files('dianna.dashboard') / 'Home.py' + args = sys.argv[1:] + + dash = files('dianna.dashboard') / 'Home.py' # https://docs.streamlit.io/library/advanced-features/configuration sys.argv = [ - 'streamlit', - 'run', - str(dashboard), - '--theme.base', - 'light', - '--theme.primaryColor', - '7030a0', - '--theme.secondaryBackgroundColor', - 'e4f3f9', - '--browser.gatherUsageStats', - 'false', + *('streamlit', 'run', str(dash)), + *('--theme.base', 'light'), + *('--theme.primaryColor', '7030a0'), + *('--theme.secondaryBackgroundColor', 'e4f3f9'), + *('--browser.gatherUsageStats', 'false'), + *args, ] sys.exit(stcli.main()) diff --git a/dianna/dashboard/Home.py b/dianna/dashboard/Home.py index fbb66de1..0f623b13 100644 --- a/dianna/dashboard/Home.py +++ b/dianna/dashboard/Home.py @@ -2,7 +2,6 @@ from _shared import add_sidebar_logo from _shared import data_directory - st.set_page_config(page_title="Dianna's dashboard", page_icon='📊', layout='centered', diff --git a/dianna/dashboard/_models_text.py b/dianna/dashboard/_models_text.py index 2cd9a148..c191b87b 100644 --- a/dianna/dashboard/_models_text.py +++ b/dianna/dashboard/_models_text.py @@ -3,7 +3,6 @@ from dianna import explain_text from dianna.utils.tokenizers import SpacyTokenizer - tokenizer = SpacyTokenizer() diff --git a/dianna/dashboard/_shared.py b/dianna/dashboard/_shared.py index cae66e6d..ce5d3966 100644 --- a/dianna/dashboard/_shared.py +++ b/dianna/dashboard/_shared.py @@ -6,13 +6,11 @@ import numpy as np import streamlit as st - if sys.version_info < (3, 10): from importlib_resources import files else: from importlib.resources import files - data_directory = files('dianna.data') diff --git a/dianna/dashboard/pages/1_Images.py b/dianna/dashboard/pages/1_Images.py index e1558954..313f265e 100644 --- a/dianna/dashboard/pages/1_Images.py +++ b/dianna/dashboard/pages/1_Images.py @@ -11,7 +11,6 @@ from _shared import data_directory from dianna.visualization import plot_image - add_sidebar_logo() st.title('Image explanation') diff --git a/dianna/dashboard/pages/2_Text.py b/dianna/dashboard/pages/2_Text.py index 5c4cf4f6..022396c6 100644 --- a/dianna/dashboard/pages/2_Text.py +++ b/dianna/dashboard/pages/2_Text.py @@ -11,7 +11,6 @@ from _shared import data_directory from _text_utils import format_word_importances - add_sidebar_logo() st.title('Text explanation') diff --git a/dianna/dashboard/pages/3_Time_series.py b/dianna/dashboard/pages/3_Time_series.py index e2ec69a7..27f7efa7 100644 --- a/dianna/dashboard/pages/3_Time_series.py +++ b/dianna/dashboard/pages/3_Time_series.py @@ -11,7 +11,6 @@ from _ts_utils import open_timeseries from dianna.visualization import plot_timeseries - add_sidebar_logo() st.title('Time series explanation') diff --git a/dianna/utils/tokenizers.py b/dianna/utils/tokenizers.py index 364dfcf8..7e62853c 100644 --- a/dianna/utils/tokenizers.py +++ b/dianna/utils/tokenizers.py @@ -4,7 +4,6 @@ from typing import List import numpy as np - try: from torchtext.data import get_tokenizer except ImportError as err: diff --git a/docs/developer_info.rst b/docs/developer_info.rst index 5dc66808..b3ab462b 100644 --- a/docs/developer_info.rst +++ b/docs/developer_info.rst @@ -86,6 +86,32 @@ The second is to use ``tox``, which must be installed separately (e.g. with ``pi Testing with ``tox`` allows for keeping the testing environment separate from your development environment. The development environment will typically accumulate (old) packages during development that interfere with testing; this problem is avoided by testing with ``tox``. +Testing the dashboard +~~~~~~~~~~~~~~~~~~~~~ + +The dashboard workflow can be tested using `playwright `__. + +Setup: + +.. code:: shell + + pip install pytest-playwright + playwright install chromium + +To run the dashboard tests: + +.. code:: shell + + pytest -v --dashboard + +To help with developing the dashboard tests, +you can use the `playwright code generator `__: + +.. code:: shell + + playwright codegen http://localhost:8501 + + Running linters locally ----------------------- diff --git a/pyproject.toml b/pyproject.toml index 3eeeb554..6a49ce16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,5 +93,4 @@ line-length = 120 [tool.ruff.isort] known-first-party = ["dianna"] force-single-line = true -lines-after-imports = 2 no-lines-before = ["future","standard-library","third-party","first-party","local-folder"] diff --git a/setup.cfg b/setup.cfg index c3de934f..962f5d0b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,7 @@ install_requires = [options.extras_require] dev = bump2version + pytest-playwright pytest pytest-cov pre-commit @@ -116,5 +117,6 @@ console_scripts = [options.packages.find] include = dianna, dianna.* -[yapf] -blank_lines_between_top_level_imports_and_variables = 2 +[tool:pytest] +markers = + dashboard: Test dashboard user workflow, requires playwright with browser installed (`playwright install chromium`) diff --git a/setup.py b/setup.py index b83abc33..3bac968b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python from setuptools import setup - # see setup.cfg setup() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..bea72b05 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +import pytest + + +def pytest_addoption(parser): + """Add options to pytest.""" + parser.addoption('--dashboard', + action='store_true', + default=False, + help='Run dashboard workflow tests') + + +def pytest_collection_modifyitems(config, items): + """Modify items for pytest.""" + if config.getoption('--dashboard'): + return + + skip_dashboard = pytest.mark.skip( + reason='Use `-m dashboard` to test dashboard workflow.') + for item in items: + if 'dashboard' in item.keywords: + item.add_marker(skip_dashboard) diff --git a/tests/methods/time_series_test_case.py b/tests/methods/time_series_test_case.py index 40e42e73..55e8a036 100644 --- a/tests/methods/time_series_test_case.py +++ b/tests/methods/time_series_test_case.py @@ -1,6 +1,5 @@ import numpy as np - """Test case for timeseries xai methods. This test case is designed to show if the xai methods could provide reasonable results. In this test case, every test instance is a 28 days by 1 channel array indicating the max temp on a day. diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py new file mode 100644 index 00000000..8606e06c --- /dev/null +++ b/tests/test_dashboard.py @@ -0,0 +1,146 @@ +"""Module to test the dashboard. + +This test module uses (playwright)[https://playwright.dev/python/] +to test the user workflow. + +Installation: + + pip install pytest-playwright + playwright install + +Code generation (https://playwright.dev/python/docs/codegen): + + playwright codegen http://localhost:8501 + +Set `LOCAL=True` to connect to local instance for debugging +""" + +import time +from contextlib import contextmanager +import pytest +from playwright.sync_api import Page +from playwright.sync_api import expect + +LOCAL = False + +PORT = '8501' if LOCAL else '8502' +BASE_URL = f'localhost:{PORT}' + +pytestmark = pytest.mark.dashboard + + +@pytest.fixture(scope='module', autouse=True) +def before_module(): + """Run dashboard in module scope.""" + with run_streamlit(): + yield + + +@contextmanager +def run_streamlit(): + """Run the dashboard.""" + import subprocess + + if not LOCAL: + p = subprocess.Popen([ + 'dianna-dashboard', + '--server.port', + PORT, + '--server.headless', + 'true', + ]) + time.sleep(5) + + yield + + if not LOCAL: + p.kill() + + +def test_page_load(page: Page): + """Test performance of landing page.""" + page.goto(BASE_URL) + + selector = page.get_by_text('Running...') + selector.wait_for(state='detached') + + expect(page).to_have_title("Dianna's dashboard") + for selector in ( + page.get_by_role('img', name='0'), + page.get_by_text('Pages'), + page.get_by_text('More information'), + ): + expect(selector).to_be_visible() + + +def test_text_page(page: Page): + """Test performance of text page.""" + page.goto(f'{BASE_URL}/Text') + + page.get_by_text('Running...').wait_for(state='detached') + + expect(page).to_have_title('Text · Streamlit') + + selector = page.get_by_text( + 'Add your input data in the left panel to continue') + + expect(selector).to_be_visible(timeout=30_000) + + page.locator('label').filter( + has_text='Load example data').locator('span').click() + + expect(page.get_by_text('Select a method to continue')).to_be_visible() + + page.locator('label').filter(has_text='RISE').locator('span').click() + + page.get_by_text('Running...').wait_for(state='detached', timeout=45_000) + + for selector in ( + page.get_by_role('heading', name='RISE').get_by_text('RISE'), + # first text + page.get_by_role('heading', + name='positive').get_by_text('positive'), + page.get_by_text( + 'The movie started out great but the ending was dissappointing' + ).first, + # second text + page.get_by_role('heading', + name='negative').get_by_text('negative'), + page.get_by_text( + 'The movie started out great but the ending was dissappointing' + ).nth(1), + ): + expect(selector).to_be_visible() + + +def test_image_page(page: Page): + """Test performance of image page.""" + page.goto(f'{BASE_URL}/Images') + + page.get_by_text('Running...').wait_for(state='detached') + + expect(page).to_have_title('Images · Streamlit') + + expect( + page.get_by_text('Add your input data in the left panel to continue') + ).to_be_visible() + + page.locator('label').filter( + has_text='Load example data').locator('span').click() + + expect(page.get_by_text('Select a method to continue')).to_be_visible() + + page.locator('label').filter(has_text='RISE').locator('span').click() + + page.get_by_text('Running...').wait_for(state='detached', timeout=45_000) + + for selector in ( + page.get_by_role('heading', name='RISE').get_by_text('RISE'), + # first image + page.get_by_role('heading', name='0').get_by_text('0'), + page.get_by_role('img', name='0').first, + # second image + page.get_by_role('heading', name='1').get_by_text('1'), + page.get_by_role('img', name='0').nth(1), + ): + expect(selector).to_be_visible() diff --git a/tests/test_text_visualization.py b/tests/test_text_visualization.py index 3fa5849a..9eae3a99 100644 --- a/tests/test_text_visualization.py +++ b/tests/test_text_visualization.py @@ -5,7 +5,6 @@ from pathlib import Path from dianna.visualization.text import highlight_text - tokenizer = re.compile(r'(\w+|\S)') diff --git a/tests/utils.py b/tests/utils.py index 02a90e62..84ea80f5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -5,7 +5,6 @@ from torchtext.vocab import Vectors from dianna.utils.tokenizers import SpacyTokenizer - _mnist_1_data = """ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0