Run examples on change (#735)

* inital draft of action to test monitoring

* cleans up monitoring workflow

* more complete draft of monitoring test action

* removes draft monitoring workflow, reorganizes existing workflows

* update internal development requirements

* turn off dry run now that we're going back to prod

* reorganize environment setup

* WIP version of example execution

* adds .secrets file from act

* handles modal serve, proper system exit, drops extra script

* updates actions, better environment setup

* handle PRs with no changed files

* add back dev dependencies for jupytext and pydantic in deploy

* reverts changes to typechecking to avoid slowdown

.github/actions/setup/action.yml

@@ -0,0 +1,36 @@
+name: setup
+
+description: Set up a Python environment for the examples.
+
+inputs:
+  version:
+    description: Which Python version to install
+    required: false
+    default: "3.11"
+  devDependencies:
+    description: Whether to skip dependencies
+    required: false
+    default: "no-skip"
+
+runs:
+  using: composite
+  steps:
+    - name: Install Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.version }}
+
+    - name: Install base packages
+      shell: bash
+      run: |
+        pip install uv
+        uv pip install --system setuptools wheel
+
+    - name: Install development Python packages
+      if: ${{ inputs.devDependencies != 'skip' }}
+      shell: bash
+      run: uv pip install --system -r internal/requirements.txt
+
+    - name: Install the modal client
+      shell: bash
+      run: uv pip install --system modal

.github/workflows/cd.yml

@@ -17,13 +17,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
-
-      - name: Install Modal client package and jupytext
-        run: pip install modal-client jupytext pydantic~=1.10
+          fetch-depth: 1
+      - uses: ./.github/actions/setup
 
       - name: Run deployment script
         run: |

.github/workflows/check.yml

@@ -13,13 +13,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
-
-      # keep version here in sync with .pre-commit-config.yaml and other modal repos
-      - run: pip install ruff==0.2.1
+          fetch-depth: 1
+      - uses: ./.github/actions/setup
 
       - run: ruff check
 
@@ -31,33 +27,24 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
-      - name: Install NbConvert
-        run: pip install jupyter nbconvert
+          fetch-depth: 1
+      - uses: ./.github/actions/setup
 
       - name: Check notebooks are cleaned
         run: |
           jupyter nbconvert --clear-output --inplace 11_notebooks/*.ipynb
-          git diff --quiet && git diff --cached --quiet || exit 1
+          git diff --quiet 11_notebooks/*.ipynb && git diff --cached --quiet 11_notebooks/*.ipynb || exit 1
 
   pytest:
     name: Pytest
     runs-on: ubuntu-20.04
 
     steps:
       - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
-
-      - name: Install dev dependencies
-        run: pip install pytest jupytext pydantic~=1.10
-
-      - name: Install the Modal client
-        run: pip install modal-client
+          fetch-depth: 1
+      - uses: ./.github/actions/setup
 
       - name: Run
         run: pytest -v .

.github/workflows/run-examples.yml

@@ -0,0 +1,76 @@
+name: Run
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**.py"
+  push:
+    branches:
+      - main
+    paths:
+      - "**.py"
+  workflow_dispatch:
+
+# Cancel previous runs of the same PR but do not cancel previous runs on main
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+env:
+  TERM: linux
+  TERMINFO: /etc/terminfo
+  MODAL_TOKEN_ID: ${{ secrets.MODAL_MODAL_LABS_TOKEN_ID }}
+  MODAL_TOKEN_SECRET: ${{ secrets.MODAL_MODAL_LABS_TOKEN_SECRET }}
+  MODAL_ENVIRONMENT: main
+
+jobs:
+  # Output all changed files in a JSON format compatible with GitHub Actions job matrices
+  diff-matrix:
+    name: Generate matrix of changed examples
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.diff.outputs.all_changed_files }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Find changed examples
+        id: diff
+        uses: tj-actions/changed-files@v44
+        with:
+          files: "**.py"
+          files_ignore: "internal/**,misc/**"
+          matrix: true
+
+      - name: List all changed examples
+        run: echo '${{ steps.diff.outputs.all_changed_files }}'
+
+  # Run each changed example, using the output of the previous step as a job matrix
+  run-changed:
+    name: Run changed example
+    needs: [diff-matrix]
+    if:
+      ${{ needs.diff-matrix.outputs.matrix != '[]' &&
+      needs.diff-matrix.outputs.matrix != '' }}
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        file: ${{ fromJson(needs.diff-matrix.outputs.matrix) }}
+      fail-fast: false
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: ./.github/actions/setup
+
+      - name: Run example
+        run: |
+          echo "Running ${{ matrix.file }}"
+          stem=$(basename "${{ matrix.file }}" .py)
+          python3 -m internal.run_example $stem || exit $?

.gitignore

@@ -3,3 +3,6 @@
 
 venv
 .venv
+
+# secrets file for act, tool for local GitHub Actions testing
+.secrets

internal/requirements.txt

@@ -1,5 +1,8 @@
-modal
 pytest
+jupyter
+ipython
+nbconvert
 jupytext~=1.16.1
 pydantic~=1.10.14
-mypy==0.950
+mypy==1.2.0
+ruff==0.2.1

internal/run_example.py

@@ -0,0 +1,50 @@
+import os
+import subprocess
+import sys
+import time
+
+from . import utils
+
+MINUTES = 60
+TIMEOUT = 12 * MINUTES
+
+
+def run_script(example):
+    t0 = time.time()
+
+    try:
+        print(f"cli args: {example.cli_args}")
+        process = subprocess.run(
+            example.cli_args,
+            env=os.environ | {"MODAL_SERVE_TIMEOUT": "5.0"},
+            timeout=TIMEOUT,
+        )
+        total_time = time.time() - t0
+        if process.returncode == 0:
+            print(f"Success after {total_time:.2f}s :)")
+        else:
+            print(
+                f"Failed after {total_time:.2f}s with return code {process.returncode} :("
+            )
+
+        returncode = process.returncode
+
+    except subprocess.TimeoutExpired:
+        print(f"Past timeout of {TIMEOUT}s :(")
+        returncode = 999
+
+    return returncode
+
+
+def run_single_example(stem):
+    examples = utils.get_examples()
+    for example in examples:
+        if stem == example.stem:
+            return run_script(example)
+    else:
+        print(f"Could not find example name {stem}")
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run_single_example(sys.argv[1]))