Merge pull request #168 from Wiebke/test-and-lint-hooks-and-actions

Add pre-commit hooks and Github action for linting and formatting

.env.example

@@ -1,30 +1,30 @@
 # The URI of a container in a Tiled server with data to be segmented
-# Replace <your-value-here> with the URI for such a container. 
+# Replace <your-value-here> with the URI for such a container.
 # Examples are:
-# the 'reconstruction' container of the public server, containing tomography reconstructions: 
+# the 'reconstruction' container of the public server, containing tomography reconstructions:
 #   'https://tiled-seg.als.lbl.gov/api/v1/metadata/reconstruction'
-# the root container of a local server: 
+# the root container of a local server:
 #   'http://localhost:8000/'
-# the 'processed' container of a server running in a docker container: 
+# the 'processed' container of a server running in a docker container:
 #   'http://tiled:8000/api/v1/metadata/processed'
 DATA_TILED_URI=<your-value-here>
 # API key for accessing the Tiled server and container from DATA_TILED_URI
 # Replace <your-value-here> with your API key
 DATA_TILED_API_KEY=<your-value-here>
 
 # The URI of a container in a Tiled server where we can store mask information
-# Replace <your-value-here> with the URL of your Tiled server. 
+# Replace <your-value-here> with the URL of your Tiled server.
 # You will need write-access to this container.
 MASK_TILED_URI=<your-value-here>
 # API key for accessing the Tiled server and container from MASK_TILED_URI
-# Replace <your-value-here> with your API key 
+# Replace <your-value-here> with your API key
 MASK_TILED_API_KEY=<api-key>
 
 # The URI of a Tiled server where segmentation results can be retrieved
 # The Tiled server will most likely be the same as the one used for storing masks
-# This top-level URI is not used directly. 
+# This top-level URI is not used directly.
 # Instead we expect ML jobs to return one or more URIs pointing to specific results
-# Replace <your-value-here> with the URL of your Tiled server. 
+# Replace <your-value-here> with the URL of your Tiled server.
 SEG_TILED_URI=<your-value-here>
 # API key for accessing the Tiled server from SEG_TILED_URI
 # Replace <your-value-here> with your API key
@@ -37,4 +37,4 @@ MODE='dev'
 
 # Basic authentication for segmentation application when deploying on a publicly accessible server
 USER_NAME=<to-be-specified-per-deployment>
-USER_PASSWORD=<to-be-specified-per-deployment>
+USER_PASSWORD=<to-be-specified-per-deployment>

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+# 127 is width of the Github code viewer,
+# black default is 88 so this will only warn about comments >127
+max-line-length = 127
+# Ignore errors due to incompatibility with black
+#https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html
+extend-ignore = E203,E701

.github/workflows/publish-image.yml

@@ -42,4 +42,4 @@ jobs:
           file: Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test-lint.yml

@@ -0,0 +1,34 @@
+name: test-and-lint
+
+on: pull_request
+
+jobs:
+  test-and-lint:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.9.16
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.9.16'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Run isort
+        uses: isort/isort-action@master
+      - name: Test formatting with black
+        run: |
+          black . --check
+      - name: Run pytest
+        run: |
+          python -m pytest

.gitignore

@@ -23,4 +23,4 @@ env
 *.tiff
 *.tif
 
-.env
+.env

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+default_language_version:
+    python: python3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-ast
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.2.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]

COPYRIGHT.txt

@@ -10,5 +10,5 @@ NOTICE.  This Software was developed under funding from the U.S. Department
 of Energy and the U.S. Government consequently retains certain rights.  As
 such, the U.S. Government has been granted for itself and others acting on
 its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
-Software to reproduce, distribute copies to the public, prepare derivative 
+Software to reproduce, distribute copies to the public, prepare derivative
 works, and perform publicly and display publicly, and to permit others to do so.

README.md

@@ -1,9 +1,9 @@
 # Dash App for Segmentation of High-Resolution Images [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1)](https://pycqa.github.io/isort/)
 
 
-This application is built using Plotly's [Dash](https://dash.plotly.com/) framework and provides a web-based interface for visualizing and annotating high resolution images output from [ALS](https://als.lbl.gov/) beamlines. 
+This application is built using Plotly's [Dash](https://dash.plotly.com/) framework and provides a web-based interface for visualizing and annotating high resolution images output from [ALS](https://als.lbl.gov/) beamlines.
 
-Image data is accessed via a [Tiled](https://github.com/bluesky/tiled) client, which provides chunkwise access to multidimensional TIFF sequences. 
+Image data is accessed via a [Tiled](https://github.com/bluesky/tiled) client, which provides chunkwise access to multidimensional TIFF sequences.
 
 ![plot](assets/preview.png)
 
@@ -17,7 +17,7 @@ Image data is accessed via a [Tiled](https://github.com/bluesky/tiled) client, w
 pip install -r requirements.txt
 ```
 
-and 
+and
 
 ```
 pip install -r requirements-dev.txt
@@ -32,7 +32,7 @@ DASH_DEPLOYMENT_LOC='Local'
 MODE='dev'
 ```
 
-3. Start a local server: 
+3. Start a local server:
 
 ```
 python app.py
@@ -43,7 +43,7 @@ python app.py
 For local testing of just the annotation functionality, developers may also choose to set up a local Tiled server with access to minimal datasets (eg. in the case that the remote server is down).
 
 To download some sample data and serve it with a local Tiled serve
-1. Additionally install the Tiled server components with `pip install "tiled[server]"`.
+1. Additionally install the Tiled server components with `pip install "tiled[server]==0.1.0a114"`.
 2. Set the input Tiled URI to localhost, e.g. set `DATA_TILED_URI`, to `http://localhost:8000/` within the `.env` file (or within your environmental variables), and use the result of a key generator (e.g. with `python3 -c "import secrets; print(secrets.token_hex(32))"`) for the key entry `DATA_TILED_API_KEY`
 2. Run the script `python3 utils/download_sample_data.py`. This will create a `data/` directory and download 2 sample projects with 2 images each.
 3. Run `/tiled_serve_dir.sh`.

app.py

@@ -4,9 +4,9 @@
 import dash_mantine_components as dmc
 from dash import Dash, dcc
 
-from callbacks.control_bar import *
-from callbacks.image_viewer import *
-from callbacks.segmentation import *
+from callbacks.control_bar import *  # noqa: F403, F401
+from callbacks.image_viewer import *  # noqa: F403, F401
+from callbacks.segmentation import *  # noqa: F403, F401
 from components.control_bar import layout as control_bar_layout
 from components.image_viewer import layout as image_viewer_layout
 

assets/functions.js

@@ -49,4 +49,4 @@ function remove_focus() {
     sliderContainer.addEventListener('blur', () => {
         sliderContainer.blur();
     });
-}
+}

assets/style.css

@@ -142,4 +142,4 @@ body {
 /* used to hide viewfinder */
 .hidden {
     display: none;
-}
+}

callbacks/control_bar.py

@@ -30,7 +30,7 @@
 from utils.plot_utils import generate_notification, generate_notification_bg_icon_col
 
 # TODO - temporary local file path and user for annotation saving and exporting
-EXPORT_FILE_PATH = os.getenv("EXPORT_FILE_PATH", "data/exported_annotation_data.json")
+EXPORT_FILE_PATH = os.getenv("EXPORT_FILE_PATH", "exported_annotation_data.json")
 USER_NAME = os.getenv("USER_NAME", "user1")
 
 # Create an empty file if it doesn't exist
@@ -58,7 +58,8 @@ def update_current_class_selection(
     edit_modal_opened,
 ):
     """
-    This callback is responsible for updating the current class selection when a class is clicked on, or when a keybind is pressed.
+    This callback is responsible for updating the current class selection when a class is clicked on,
+    or when a keybind is pressed.
     """
     current_selection = None
     label_name = None
@@ -757,7 +758,9 @@ def populate_load_annotations_dropdown_menu_options(modal_opened, image_src):
     if not modal_opened:
         raise PreventUpdate
 
-    data = tiled_dataset.DEV_load_exported_json_data(EXPORT_FILE_PATH, USER_NAME, image_src)
+    data = tiled_dataset.DEV_load_exported_json_data(
+        EXPORT_FILE_PATH, USER_NAME, image_src
+    )
     if not data:
         return "No annotations found for the selected data source."
 
@@ -801,8 +804,12 @@ def load_and_apply_selected_annotations(selected_annotation, image_src, img_idx)
     )["index"]
 
     # TODO : when quering from the server, load (data) for user, source, time
-    data = tiled_dataset.DEV_load_exported_json_data(EXPORT_FILE_PATH, USER_NAME, image_src)
-    data = tiled_dataset.DEV_filter_json_data_by_timestamp(data, str(selected_annotation_timestamp))
+    data = tiled_dataset.DEV_load_exported_json_data(
+        EXPORT_FILE_PATH, USER_NAME, image_src
+    )
+    data = tiled_dataset.DEV_filter_json_data_by_timestamp(
+        data, str(selected_annotation_timestamp)
+    )
     data = data[0]["data"]
 
     annotations = []
@@ -847,8 +854,10 @@ def populate_classification_results(
 ):
     if refresh_tiled:
         tiled_dataset.refresh_data()
-
-    data_options = [ item for item in tiled_dataset.get_data_project_names() if "seg" not in item] 
+
+    data_options = [
+        item for item in tiled_dataset.get_data_project_names() if "seg" not in item
+    ]
     results = []
     value = None
     checked = False
@@ -875,7 +884,15 @@ def populate_classification_results(
             disabled_toggle = False
             disabled_slider = False
 
-    return results, value, disabled_dropdown, checked, disabled_toggle, disabled_slider, data_options
+    return (
+        results,
+        value,
+        disabled_dropdown,
+        checked,
+        disabled_toggle,
+        disabled_slider,
+        data_options,
+    )
 
 
 @callback(

callbacks/image_viewer.py

@@ -1,7 +1,4 @@
-import random
-
 import dash
-import dash_mantine_components as dmc
 import numpy as np
 import plotly.express as px
 import plotly.graph_objects as go
@@ -17,8 +14,6 @@
     ctx,
 )
 from dash.exceptions import PreventUpdate
-from dash_iconify import DashIconify
-from plotly.subplots import make_subplots
 
 from constants import ANNOT_ICONS, ANNOT_NOTIFICATION_MSGS, KEYBINDS
 from utils.data_utils import tiled_dataset
@@ -123,7 +118,9 @@ def render_image(
             ):
                 return [dash.no_update] * 7 + ["hidden"]
             if str(image_idx + 1) in tiled_dataset.get_annotated_segmented_results():
-                result = tiled_dataset.get_data_sequence_by_name(seg_result_selection)[image_idx]
+                result = tiled_dataset.get_data_sequence_by_name(seg_result_selection)[
+                    image_idx
+                ]
             else:
                 result = None
     else:
@@ -298,7 +295,8 @@ def update_viewfinder(relayout_data, annotation_store):
     contained within the figure layout, so that if the user zooms out/pans away,
     they will still be able to see the viewfinder box.
     """
-    # Callback is triggered when the image is first loaded, but the annotation_store is not yet populated so we need to prevent the update
+    # Callback is triggered when the image is first loaded, but the annotation_store is not yet populated
+    # so we need to prevent the update
     if not annotation_store["active_img_shape"]:
         raise PreventUpdate
     patched_fig = Patch()
@@ -486,7 +484,9 @@ def update_slider_values(project_name, annotation_store):
     "update_selection_and_image" callback which will update image and slider selection component.
     """
     # Retrieve data shape if project_name is valid and points to a 3d array
-    data_shape = tiled_dataset.get_data_shape_by_name(project_name) if project_name else None
+    data_shape = (
+        tiled_dataset.get_data_shape_by_name(project_name) if project_name else None
+    )
     disable_slider = data_shape is None
     if not disable_slider:
         # TODO: Assuming that all slices have the same image shape

callbacks/segmentation.py

@@ -3,13 +3,11 @@
 import uuid
 
 import dash_mantine_components as dmc
-import numpy as np
 import requests
 from dash import ALL, Input, Output, State, callback, no_update
 from dash.exceptions import PreventUpdate
 
 from utils.data_utils import tiled_dataset
-from utils.annotations import Annotations
 
 MODE = os.getenv("MODE", "")
 
@@ -24,7 +22,7 @@
             "job_kwargs": {
                 "uri": "mlexchange1/random-forest-dc:1.1",
                 "type": "docker",
-                "cmd": 'python random_forest.py data/seg-results/spiral/image-train data/seg-results-test/spiral/feature data/seg-results/spiral/mask data/seg-results-test/spiral/model \'{"n_estimators": 30, "oob_score": true, "max_depth": 8}\'',
+                "cmd": 'python random_forest.py data/seg-results/spiral/image-train data/seg-results-test/spiral/feature data/seg-results/spiral/mask data/seg-results-test/spiral/model \'{"n_estimators": 30, "oob_score": true, "max_depth": 8}\'',  # noqa: E501
                 "kwargs": {
                     "job_type": "train",
                     "experiment_id": "123",
@@ -41,7 +39,7 @@
             "job_kwargs": {
                 "uri": "mlexchange1/random-forest-dc:1.1",
                 "type": "docker",
-                "cmd": "python segment.py data/data/20221222_085501_looking_from_above_spiralUP_CounterClockwise_endPointAtDoor_0-1000 data/seg-results-test/spiral/model/random-forest.model data/seg-results-test/spiral/output '{\"show_progress\": 1}'",
+                "cmd": "python segment.py data/data/20221222_085501_looking_from_above_spiralUP_CounterClockwise_endPointAtDoor_0-1000 data/seg-results-test/spiral/model/random-forest.model data/seg-results-test/spiral/output '{\"show_progress\": 1}'",  # noqa: E501
                 "kwargs": {
                     "job_type": "train",
                     "experiment_id": "124",

components/annotation_class.py

@@ -176,7 +176,8 @@ def annotation_class_item(class_color, class_label, existing_ids, data=None):
                 children=[
                     dmc.Center(
                         dmc.Text(
-                            "This action will permanently clear all annotations from this class. Are you sure you want to proceed?",
+                            "This action will permanently clear all annotations from this class."
+                            + " Are you sure you want to proceed?",
                         )
                     ),
                     dmc.Space(h=10),