diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 612d7a24..d918b596 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -21,11 +21,14 @@ jobs:
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@v3
+
- uses: ./.github/actions/install-python-and-package
with:
python-version: '3.10'
+
- name: Run unit tests
run: pytest -v
+
- name: Verify that we can build the package
run: python3 setup.py sdist bdist_wheel
@@ -40,10 +43,30 @@ jobs:
python-version: ['3.7', '3.8', '3.9', '3.10']
steps:
- uses: actions/checkout@v3
+
- uses: ./.github/actions/install-python-and-package
with:
python-version: ${{ matrix.python-version }}
+
- name: Run unit tests
run: pytest -v
+
- name: Verify that we can build the package
run: python3 setup.py sdist bdist_wheel
+
+ test_dashboard:
+ name: Test dashboard
+ if: github.event.pull_request.draft == false
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: ./.github/actions/install-python-and-package
+ with:
+ python-version: '3.10'
+ extras-require: dev,dashboard
+
+ - name: Ensure browser is installed
+ run: python -m playwright install chromium
+
+ - name: Test dashboard
+ run: pytest -v -m dashboard --dashboard
diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
index 10019413..cb236036 100644
--- a/.github/workflows/sonarcloud.yml
+++ b/.github/workflows/sonarcloud.yml
@@ -24,11 +24,15 @@ jobs:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+
- uses: ./.github/actions/install-python-and-package
+
- name: Run unit tests with coverage
- run: pytest --cov --cov-report term --cov-report xml --cov-report html --junitxml=xunit-result.xml tests/
+ run: pytest --cov --cov-report term --cov-report xml --cov-report html --junitxml=xunit-result.xml tests/ -m "not dashboard"
+
- name: Correct coverage paths
run: sed -i "s+$PWD/++g" coverage.xml
+
- name: SonarCloud Scan
uses: SonarSource/sonarcloud-github-action@master
env:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 48cad4d1..0bedf2b4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -23,7 +23,7 @@ repos:
additional_dependencies:
- toml
- repo: https://github.com/charliermarsh/ruff-pre-commit
- rev: 'v0.0.254'
+ rev: 'v0.0.264'
hooks:
- id: ruff
args: [--fix]
diff --git a/dianna/__init__.py b/dianna/__init__.py
index 6cc2b792..70f0cdf3 100644
--- a/dianna/__init__.py
+++ b/dianna/__init__.py
@@ -24,7 +24,6 @@
import logging
from . import utils
-
logging.getLogger(__name__).addHandler(logging.NullHandler())
__author__ = "DIANNA Team"
diff --git a/dianna/cli.py b/dianna/cli.py
index f805ec33..4c3e1977 100644
--- a/dianna/cli.py
+++ b/dianna/cli.py
@@ -1,6 +1,5 @@
import sys
-
if sys.version_info < (3, 10):
from importlib_resources import files
else:
@@ -11,21 +10,18 @@ def dashboard():
"""Start streamlit dashboard."""
from streamlit.web import cli as stcli
- dashboard = files('dianna.dashboard') / 'Home.py'
+ args = sys.argv[1:]
+
+ dash = files('dianna.dashboard') / 'Home.py'
# https://docs.streamlit.io/library/advanced-features/configuration
sys.argv = [
- 'streamlit',
- 'run',
- str(dashboard),
- '--theme.base',
- 'light',
- '--theme.primaryColor',
- '7030a0',
- '--theme.secondaryBackgroundColor',
- 'e4f3f9',
- '--browser.gatherUsageStats',
- 'false',
+ *('streamlit', 'run', str(dash)),
+ *('--theme.base', 'light'),
+ *('--theme.primaryColor', '7030a0'),
+ *('--theme.secondaryBackgroundColor', 'e4f3f9'),
+ *('--browser.gatherUsageStats', 'false'),
+ *args,
]
sys.exit(stcli.main())
diff --git a/dianna/dashboard/Home.py b/dianna/dashboard/Home.py
index fbb66de1..0f623b13 100644
--- a/dianna/dashboard/Home.py
+++ b/dianna/dashboard/Home.py
@@ -2,7 +2,6 @@
from _shared import add_sidebar_logo
from _shared import data_directory
-
st.set_page_config(page_title="Dianna's dashboard",
page_icon='📊',
layout='centered',
diff --git a/dianna/dashboard/_models_text.py b/dianna/dashboard/_models_text.py
index 2cd9a148..c191b87b 100644
--- a/dianna/dashboard/_models_text.py
+++ b/dianna/dashboard/_models_text.py
@@ -3,7 +3,6 @@
from dianna import explain_text
from dianna.utils.tokenizers import SpacyTokenizer
-
tokenizer = SpacyTokenizer()
diff --git a/dianna/dashboard/_shared.py b/dianna/dashboard/_shared.py
index cae66e6d..ce5d3966 100644
--- a/dianna/dashboard/_shared.py
+++ b/dianna/dashboard/_shared.py
@@ -6,13 +6,11 @@
import numpy as np
import streamlit as st
-
if sys.version_info < (3, 10):
from importlib_resources import files
else:
from importlib.resources import files
-
data_directory = files('dianna.data')
diff --git a/dianna/dashboard/pages/1_Images.py b/dianna/dashboard/pages/1_Images.py
index e1558954..313f265e 100644
--- a/dianna/dashboard/pages/1_Images.py
+++ b/dianna/dashboard/pages/1_Images.py
@@ -11,7 +11,6 @@
from _shared import data_directory
from dianna.visualization import plot_image
-
add_sidebar_logo()
st.title('Image explanation')
diff --git a/dianna/dashboard/pages/2_Text.py b/dianna/dashboard/pages/2_Text.py
index 5c4cf4f6..022396c6 100644
--- a/dianna/dashboard/pages/2_Text.py
+++ b/dianna/dashboard/pages/2_Text.py
@@ -11,7 +11,6 @@
from _shared import data_directory
from _text_utils import format_word_importances
-
add_sidebar_logo()
st.title('Text explanation')
diff --git a/dianna/dashboard/pages/3_Time_series.py b/dianna/dashboard/pages/3_Time_series.py
index e2ec69a7..27f7efa7 100644
--- a/dianna/dashboard/pages/3_Time_series.py
+++ b/dianna/dashboard/pages/3_Time_series.py
@@ -11,7 +11,6 @@
from _ts_utils import open_timeseries
from dianna.visualization import plot_timeseries
-
add_sidebar_logo()
st.title('Time series explanation')
diff --git a/dianna/utils/tokenizers.py b/dianna/utils/tokenizers.py
index 364dfcf8..7e62853c 100644
--- a/dianna/utils/tokenizers.py
+++ b/dianna/utils/tokenizers.py
@@ -4,7 +4,6 @@
from typing import List
import numpy as np
-
try:
from torchtext.data import get_tokenizer
except ImportError as err:
diff --git a/docs/developer_info.rst b/docs/developer_info.rst
index 5dc66808..b3ab462b 100644
--- a/docs/developer_info.rst
+++ b/docs/developer_info.rst
@@ -86,6 +86,32 @@ The second is to use ``tox``, which must be installed separately (e.g. with ``pi
Testing with ``tox`` allows for keeping the testing environment separate from your development environment.
The development environment will typically accumulate (old) packages during development that interfere with testing; this problem is avoided by testing with ``tox``.
+Testing the dashboard
+~~~~~~~~~~~~~~~~~~~~~
+
+The dashboard workflow can be tested using `playwright `__.
+
+Setup:
+
+.. code:: shell
+
+ pip install pytest-playwright
+ playwright install chromium
+
+To run the dashboard tests:
+
+.. code:: shell
+
+ pytest -v --dashboard
+
+To help with developing the dashboard tests,
+you can use the `playwright code generator `__:
+
+.. code:: shell
+
+ playwright codegen http://localhost:8501
+
+
Running linters locally
-----------------------
diff --git a/pyproject.toml b/pyproject.toml
index 3eeeb554..6a49ce16 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -93,5 +93,4 @@ line-length = 120
[tool.ruff.isort]
known-first-party = ["dianna"]
force-single-line = true
-lines-after-imports = 2
no-lines-before = ["future","standard-library","third-party","first-party","local-folder"]
diff --git a/setup.cfg b/setup.cfg
index c3de934f..962f5d0b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -62,6 +62,7 @@ install_requires =
[options.extras_require]
dev =
bump2version
+ pytest-playwright
pytest
pytest-cov
pre-commit
@@ -116,5 +117,6 @@ console_scripts =
[options.packages.find]
include = dianna, dianna.*
-[yapf]
-blank_lines_between_top_level_imports_and_variables = 2
+[tool:pytest]
+markers =
+ dashboard: Test dashboard user workflow, requires playwright with browser installed (`playwright install chromium`)
diff --git a/setup.py b/setup.py
index b83abc33..3bac968b 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,5 @@
#!/usr/bin/env python
from setuptools import setup
-
# see setup.cfg
setup()
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 00000000..bea72b05
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,21 @@
+import pytest
+
+
+def pytest_addoption(parser):
+ """Add options to pytest."""
+ parser.addoption('--dashboard',
+ action='store_true',
+ default=False,
+ help='Run dashboard workflow tests')
+
+
+def pytest_collection_modifyitems(config, items):
+ """Modify items for pytest."""
+ if config.getoption('--dashboard'):
+ return
+
+ skip_dashboard = pytest.mark.skip(
+ reason='Use `-m dashboard` to test dashboard workflow.')
+ for item in items:
+ if 'dashboard' in item.keywords:
+ item.add_marker(skip_dashboard)
diff --git a/tests/methods/time_series_test_case.py b/tests/methods/time_series_test_case.py
index 40e42e73..55e8a036 100644
--- a/tests/methods/time_series_test_case.py
+++ b/tests/methods/time_series_test_case.py
@@ -1,6 +1,5 @@
import numpy as np
-
"""Test case for timeseries xai methods.
This test case is designed to show if the xai methods could provide reasonable results.
In this test case, every test instance is a 28 days by 1 channel array indicating the max temp on a day.
diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py
new file mode 100644
index 00000000..8606e06c
--- /dev/null
+++ b/tests/test_dashboard.py
@@ -0,0 +1,146 @@
+"""Module to test the dashboard.
+
+This test module uses (playwright)[https://playwright.dev/python/]
+to test the user workflow.
+
+Installation:
+
+ pip install pytest-playwright
+ playwright install
+
+Code generation (https://playwright.dev/python/docs/codegen):
+
+ playwright codegen http://localhost:8501
+
+Set `LOCAL=True` to connect to local instance for debugging
+"""
+
+import time
+from contextlib import contextmanager
+import pytest
+from playwright.sync_api import Page
+from playwright.sync_api import expect
+
+LOCAL = False
+
+PORT = '8501' if LOCAL else '8502'
+BASE_URL = f'localhost:{PORT}'
+
+pytestmark = pytest.mark.dashboard
+
+
+@pytest.fixture(scope='module', autouse=True)
+def before_module():
+ """Run dashboard in module scope."""
+ with run_streamlit():
+ yield
+
+
+@contextmanager
+def run_streamlit():
+ """Run the dashboard."""
+ import subprocess
+
+ if not LOCAL:
+ p = subprocess.Popen([
+ 'dianna-dashboard',
+ '--server.port',
+ PORT,
+ '--server.headless',
+ 'true',
+ ])
+ time.sleep(5)
+
+ yield
+
+ if not LOCAL:
+ p.kill()
+
+
+def test_page_load(page: Page):
+ """Test performance of landing page."""
+ page.goto(BASE_URL)
+
+ selector = page.get_by_text('Running...')
+ selector.wait_for(state='detached')
+
+ expect(page).to_have_title("Dianna's dashboard")
+ for selector in (
+ page.get_by_role('img', name='0'),
+ page.get_by_text('Pages'),
+ page.get_by_text('More information'),
+ ):
+ expect(selector).to_be_visible()
+
+
+def test_text_page(page: Page):
+ """Test performance of text page."""
+ page.goto(f'{BASE_URL}/Text')
+
+ page.get_by_text('Running...').wait_for(state='detached')
+
+ expect(page).to_have_title('Text · Streamlit')
+
+ selector = page.get_by_text(
+ 'Add your input data in the left panel to continue')
+
+ expect(selector).to_be_visible(timeout=30_000)
+
+ page.locator('label').filter(
+ has_text='Load example data').locator('span').click()
+
+ expect(page.get_by_text('Select a method to continue')).to_be_visible()
+
+ page.locator('label').filter(has_text='RISE').locator('span').click()
+
+ page.get_by_text('Running...').wait_for(state='detached', timeout=45_000)
+
+ for selector in (
+ page.get_by_role('heading', name='RISE').get_by_text('RISE'),
+ # first text
+ page.get_by_role('heading',
+ name='positive').get_by_text('positive'),
+ page.get_by_text(
+ 'The movie started out great but the ending was dissappointing'
+ ).first,
+ # second text
+ page.get_by_role('heading',
+ name='negative').get_by_text('negative'),
+ page.get_by_text(
+ 'The movie started out great but the ending was dissappointing'
+ ).nth(1),
+ ):
+ expect(selector).to_be_visible()
+
+
+def test_image_page(page: Page):
+ """Test performance of image page."""
+ page.goto(f'{BASE_URL}/Images')
+
+ page.get_by_text('Running...').wait_for(state='detached')
+
+ expect(page).to_have_title('Images · Streamlit')
+
+ expect(
+ page.get_by_text('Add your input data in the left panel to continue')
+ ).to_be_visible()
+
+ page.locator('label').filter(
+ has_text='Load example data').locator('span').click()
+
+ expect(page.get_by_text('Select a method to continue')).to_be_visible()
+
+ page.locator('label').filter(has_text='RISE').locator('span').click()
+
+ page.get_by_text('Running...').wait_for(state='detached', timeout=45_000)
+
+ for selector in (
+ page.get_by_role('heading', name='RISE').get_by_text('RISE'),
+ # first image
+ page.get_by_role('heading', name='0').get_by_text('0'),
+ page.get_by_role('img', name='0').first,
+ # second image
+ page.get_by_role('heading', name='1').get_by_text('1'),
+ page.get_by_role('img', name='0').nth(1),
+ ):
+ expect(selector).to_be_visible()
diff --git a/tests/test_text_visualization.py b/tests/test_text_visualization.py
index 3fa5849a..9eae3a99 100644
--- a/tests/test_text_visualization.py
+++ b/tests/test_text_visualization.py
@@ -5,7 +5,6 @@
from pathlib import Path
from dianna.visualization.text import highlight_text
-
tokenizer = re.compile(r'(\w+|\S)')
diff --git a/tests/utils.py b/tests/utils.py
index 02a90e62..84ea80f5 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -5,7 +5,6 @@
from torchtext.vocab import Vectors
from dianna.utils.tokenizers import SpacyTokenizer
-
_mnist_1_data = """
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0