Test user workflow in dashboard (#581)

* Add test for dashboard
* Add playwright to dev dependencies
* Improve test compatibility with chromium
* Add github action for dashboard test
* Move dashboard test to its own workflow
* Do not run dashboard workflow test by default
* Run dashboard tests when requested via --dashboard
* Add some documentation

.github/workflows/build.yml

@@ -21,11 +21,14 @@ jobs:
     runs-on: 'ubuntu-latest'
     steps:
       - uses: actions/checkout@v3
+
       - uses: ./.github/actions/install-python-and-package
         with:
           python-version: '3.10'
+
       - name: Run unit tests
         run: pytest -v
+
       - name: Verify that we can build the package
         run: python3 setup.py sdist bdist_wheel
 
@@ -40,10 +43,30 @@ jobs:
         python-version: ['3.7', '3.8', '3.9', '3.10']
     steps:
       - uses: actions/checkout@v3
+
       - uses: ./.github/actions/install-python-and-package
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Run unit tests
         run: pytest -v
+
       - name: Verify that we can build the package
         run: python3 setup.py sdist bdist_wheel
+
+  test_dashboard:
+    name: Test dashboard
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/install-python-and-package
+        with:
+          python-version: '3.10'
+          extras-require: dev,dashboard
+
+      - name: Ensure browser is installed
+        run: python -m playwright install chromium
+
+      - name: Test dashboard
+        run: pytest -v -m dashboard --dashboard

.github/workflows/sonarcloud.yml

@@ -24,11 +24,15 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+
       - uses: ./.github/actions/install-python-and-package
+
       - name: Run unit tests with coverage
-        run: pytest --cov --cov-report term --cov-report xml --cov-report html --junitxml=xunit-result.xml tests/
+        run: pytest --cov --cov-report term --cov-report xml --cov-report html --junitxml=xunit-result.xml tests/ -m "not dashboard"
+
       - name: Correct coverage paths
         run: sed -i "s+$PWD/++g" coverage.xml
+
       - name: SonarCloud Scan
         uses: SonarSource/sonarcloud-github-action@master
         env:

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
         additional_dependencies:
           - toml
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.254'
+    rev: 'v0.0.264'
     hooks:
       - id: ruff
         args: [--fix]

dianna/__init__.py

@@ -24,7 +24,6 @@
 import logging
 from . import utils
 
-
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 __author__ = "DIANNA Team"

dianna/cli.py

@@ -1,6 +1,5 @@
 import sys
 
-
 if sys.version_info < (3, 10):
     from importlib_resources import files
 else:
@@ -11,21 +10,18 @@ def dashboard():
     """Start streamlit dashboard."""
     from streamlit.web import cli as stcli
 
-    dashboard = files('dianna.dashboard') / 'Home.py'
+    args = sys.argv[1:]
+
+    dash = files('dianna.dashboard') / 'Home.py'
 
     # https://docs.streamlit.io/library/advanced-features/configuration
     sys.argv = [
-        'streamlit',
-        'run',
-        str(dashboard),
-        '--theme.base',
-        'light',
-        '--theme.primaryColor',
-        '7030a0',
-        '--theme.secondaryBackgroundColor',
-        'e4f3f9',
-        '--browser.gatherUsageStats',
-        'false',
+        *('streamlit', 'run', str(dash)),
+        *('--theme.base', 'light'),
+        *('--theme.primaryColor', '7030a0'),
+        *('--theme.secondaryBackgroundColor', 'e4f3f9'),
+        *('--browser.gatherUsageStats', 'false'),
+        *args,
     ]
 
     sys.exit(stcli.main())

dianna/dashboard/Home.py

@@ -2,7 +2,6 @@
 from _shared import add_sidebar_logo
 from _shared import data_directory
 
-
 st.set_page_config(page_title="Dianna's dashboard",
                    page_icon='📊',
                    layout='centered',

dianna/dashboard/_models_text.py

@@ -3,7 +3,6 @@
 from dianna import explain_text
 from dianna.utils.tokenizers import SpacyTokenizer
 
-
 tokenizer = SpacyTokenizer()
 
 

dianna/dashboard/_shared.py

@@ -6,13 +6,11 @@
 import numpy as np
 import streamlit as st
 
-
 if sys.version_info < (3, 10):
     from importlib_resources import files
 else:
     from importlib.resources import files
 
-
 data_directory = files('dianna.data')
 
 

dianna/dashboard/pages/1_Images.py

@@ -11,7 +11,6 @@
 from _shared import data_directory
 from dianna.visualization import plot_image
 
-
 add_sidebar_logo()
 
 st.title('Image explanation')

dianna/dashboard/pages/2_Text.py

@@ -11,7 +11,6 @@
 from _shared import data_directory
 from _text_utils import format_word_importances
 
-
 add_sidebar_logo()
 
 st.title('Text explanation')

dianna/dashboard/pages/3_Time_series.py

@@ -11,7 +11,6 @@
 from _ts_utils import open_timeseries
 from dianna.visualization import plot_timeseries
 
-
 add_sidebar_logo()
 
 st.title('Time series explanation')

dianna/utils/tokenizers.py

@@ -4,7 +4,6 @@
 from typing import List
 import numpy as np
 
-
 try:
     from torchtext.data import get_tokenizer
 except ImportError as err:

docs/developer_info.rst

@@ -86,6 +86,32 @@ The second is to use ``tox``, which must be installed separately (e.g. with ``pi
 Testing with ``tox`` allows for keeping the testing environment separate from your development environment.
 The development environment will typically accumulate (old) packages during development that interfere with testing; this problem is avoided by testing with ``tox``.
 
+Testing the dashboard
+~~~~~~~~~~~~~~~~~~~~~
+
+The dashboard workflow can be tested using `playwright <https://playwright.dev/python/>`__.
+
+Setup:
+
+.. code:: shell
+
+    pip install pytest-playwright
+    playwright install chromium
+
+To run the dashboard tests:
+
+.. code:: shell
+
+    pytest -v --dashboard
+
+To help with developing the dashboard tests,
+you can use the `playwright code generator <https://playwright.dev/python/docs/codegen)>`__:
+
+.. code:: shell
+
+    playwright codegen http://localhost:8501
+
+
 Running linters locally
 -----------------------
 

pyproject.toml

@@ -93,5 +93,4 @@ line-length = 120
 [tool.ruff.isort]
 known-first-party = ["dianna"]
 force-single-line = true
-lines-after-imports = 2
 no-lines-before = ["future","standard-library","third-party","first-party","local-folder"]

setup.cfg

@@ -62,6 +62,7 @@ install_requires =
 [options.extras_require]
 dev =
     bump2version
+    pytest-playwright
     pytest
     pytest-cov
     pre-commit
@@ -116,5 +117,6 @@ console_scripts =
 [options.packages.find]
 include = dianna, dianna.*
 
-[yapf]
-blank_lines_between_top_level_imports_and_variables = 2
+[tool:pytest]
+markers =
+    dashboard: Test dashboard user workflow, requires playwright with browser installed (`playwright install chromium`)

setup.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 from setuptools import setup
 
-
 # see setup.cfg
 setup()

tests/conftest.py

@@ -0,0 +1,21 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    """Add options to pytest."""
+    parser.addoption('--dashboard',
+                     action='store_true',
+                     default=False,
+                     help='Run dashboard workflow tests')
+
+
+def pytest_collection_modifyitems(config, items):
+    """Modify items for pytest."""
+    if config.getoption('--dashboard'):
+        return
+
+    skip_dashboard = pytest.mark.skip(
+        reason='Use `-m dashboard` to test dashboard workflow.')
+    for item in items:
+        if 'dashboard' in item.keywords:
+            item.add_marker(skip_dashboard)

tests/methods/time_series_test_case.py

@@ -1,6 +1,5 @@
 import numpy as np
 
-
 """Test case for timeseries xai methods.
 This test case is designed to show if the xai methods could provide reasonable results.
 In this test case, every test instance is a 28 days by 1 channel array indicating the max temp on a day.