From 96ee4f27c11b9493b5583fbec58b436bda001d9a Mon Sep 17 00:00:00 2001
From: "M.Gough" <60232355+mgough-970@users.noreply.github.com>
Date: Wed, 1 May 2024 09:32:02 -0400
Subject: [PATCH] Update ci_html_build.yml

---
 .github/workflows/ci_html_build.yml | 132 +++++++++++++++++++++++++---
 1 file changed, 122 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci_html_build.yml b/.github/workflows/ci_html_build.yml
index 834991ccb..9a4358131 100644
--- a/.github/workflows/ci_html_build.yml
+++ b/.github/workflows/ci_html_build.yml
@@ -1,15 +1,127 @@
 name: Build HTML on merge
-on:
-  pull_request:
-   branches:
-   - main
-   types: [closed]
 
-  workflow_dispatch:
+on:
+  pull_request_target:
+    types: [closed]
+    paths:
+      - '**.ipynb'
 
+env:
+  CASJOBS_PW: ${{ secrets.CASJOBS_PW }}
+  CASJOBS_USERID: ${{ secrets.CASJOBS_USERID }}
 jobs:
-  Generate_HTML:
+  prepare_matrix:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - id: set-matrix
+        run: |
+          files=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} '*.ipynb')
+          #files_json=$(echo $files | jq -R -s -c 'split("\n")[:-1]')
+          files_json=$(echo "$files" | jq -R -s -c 'split("\n")[:-1]')
+          echo "::set-output name=matrix::${files_json}"
+
+  execute-notebooks:
+    if: github.event.pull_request.merged == true
+    needs: prepare_matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        notebook: ${{fromJson(needs.prepare_matrix.outputs.matrix)}}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v3
+        with:
+          python-version:  ${{ vars.PYTHON_VERSION }}
+
+      - name: Install Dependencies (if requirements.txt exists) and Execute Notebook
+        run: |
+          notebook="${{ matrix.notebook }}"
+          dir=$(dirname "$notebook")
+          if [ -f "$dir/requirements.txt" ]; then
+            pip install -r "$dir/requirements.txt"
+          fi
+          pip install notebook
+          jupyter nbconvert --to notebook --execute --inplace "$notebook" 
+
+      - name: Commit modified file on current branch
+        run: |
+          git config user.name 'CI Bot'
+          git config user.email 'action@github.com'
+          git add "${{ matrix.notebook }}"
+          git commit -m "Storing executed notebook ${{ matrix.notebook }}"
+
+      - name: Checkout only the file to the target branch
+        run: |
+          git fetch
+          git pull
+          git checkout -f gh-storage
+          git checkout @{-1} "${{ matrix.notebook }}"
+
+      - name: Commit and push modifications to target branch
+        run: |
+          git commit -m "Storing executed notebook ${{ matrix.notebook }}"
+
+          MAX_RETRIES=5
+          RETRY_DELAY=10s
+          for i in $(seq 1 $MAX_RETRIES); do
+            git push origin gh-storage --force && break || {
+              echo "Push $i failed... waiting $RETRY_DELAY"
+              sleep $RETRY_DELAY
+            }
+          done
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+
+  generate_html:
     if: github.event.pull_request.merged == true
-    uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_builder.yml@v3
-    with:
-      python-version: ${{ vars.PYTHON_VERSION }}
+    needs: execute-notebooks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python  ${{ vars.PYTHON_VERSION }}
+      uses: actions/setup-python@v5
+      with:
+        python-version:  ${{ vars.PYTHON_VERSION }}
+        
+    - name: Install dependencies
+      run: |
+        pip install ghp-import
+        pip install jupyter-book==v0.15.1
+        pip install myst-nb
+        pip install astroid
+        pip install nbval
+        #pip install bs4
+        #pip install lxml
+        ## test to bypass the jupyter-book lower version
+        pip install jsonschema==4.6.0
+        PATH="${PATH}:${HOME}/.local/bin"
+
+    - name: Build HTML
+      run: |
+        git fetch
+        git checkout origin/gh-storage -- notebooks/
+        jupyter-book build .
+
+    # Push the book's HTML to github-pages
+    - name: GitHub Pages action
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./_build/html