Merge branch 'master' into initial-final-graph

.codecov.yml

@@ -17,18 +17,19 @@ coverage:
     project:
       default:
         # commits below this threshold will be marked as failed
-        target: '85%'
+        target: '92%'
         # how much we allow the coverage to drop
         threshold: '2%'
     patch:
       default:
-        target: '95%'
+        target: '97%'
         threshold: '5%'
 
 # files to ignore
 ignore:
   - "tests/**"
   - "ws_messages_pb2.py"
+  - "cylc/flow/scripts/report_timings.py"
 
 flag_management:
   default_rules:

.coveragerc

@@ -16,6 +16,7 @@ omit =
     tests/*
     */cylc/flow/*_pb2.py
     cylc/flow/etc/*
+    cylc/flow/scripts/report_timings.py
 parallel = True
 source = ./cylc
 timid = False

.github/workflows/1_create_release_pr.yml

@@ -21,7 +21,7 @@ jobs:
         uses: cylc/release-actions/stage-1/sanitize-inputs@v1
 
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.BASE_REF }}
           fetch-depth: 0  # need to fetch all commits to check contributors
@@ -30,7 +30,7 @@ jobs:
         uses: cylc/release-actions/check-shortlog@v1
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 

.github/workflows/2_auto_publish_release.yml

@@ -22,12 +22,12 @@ jobs:
     steps:
 
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ env.MERGE_SHA }}
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
 
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.8.10
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/bash.yml

@@ -36,6 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     strategy:
+      fail-fast: false
       matrix:
         bash-version:
           - '3.2'
@@ -44,7 +45,7 @@ jobs:
           - '5.0'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Docker container
         run: |
@@ -85,7 +86,8 @@ jobs:
 
       - name: Upload artifact
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: Upload cylc-run artifact
+          name: 'cylc-run (bash-${{ matrix.bash-version }})'
           path: cylc-run
+          include-hidden-files: true

.github/workflows/branch_sync.yml

@@ -14,109 +14,8 @@ on:
 
 jobs:
   sync:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    env:
-      BASE_BRANCH: master
-      HEAD_BRANCH: ${{ inputs.head_branch || github.ref_name }}
-      STATUS_JSON: https://raw.githubusercontent.com/cylc/cylc-admin/master/docs/status/branches.json
-      FORCE_COLOR: 2
-    steps:
-      - name: Check branch name
-        shell: python
-        run: |
-          import os
-          import json
-          import sys
-          from urllib.request import urlopen
-
-          if os.environ['GITHUB_EVENT_NAME'] == 'schedule':
-            # Get branch from status page
-            meta = json.loads(
-              urlopen(os.environ['STATUS_JSON']).read()
-            )['meta_releases']
-            version = min(meta)
-            branch = meta[version][os.environ['GITHUB_REPOSITORY']]
-          else:
-            branch = os.environ['HEAD_BRANCH'].strip()
-
-          if branch.endswith('-sync'):
-            sys.exit("::error::Do not run this workflow for already-created sync branches")
-
-          with open(os.environ['GITHUB_ENV'], 'a') as F:
-            print(f'HEAD_BRANCH={branch}', file=F)
-            print(f'SYNC_BRANCH={branch}-sync', file=F)
-
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: master
-
-      - name: Configure git
-        uses: cylc/release-actions/configure-git@v1
-
-      - name: Checkout sync branch if it exists
-        continue-on-error: true
-        run: |
-          git switch -c "$SYNC_BRANCH" "origin/${SYNC_BRANCH}"
-          echo "BASE_BRANCH=$SYNC_BRANCH" >> "$GITHUB_ENV"
-
-      - name: Attempt fast-forward
-        id: ff
-        run: |
-          if git merge "origin/${HEAD_BRANCH}" --ff-only; then
-            if [[ "$(git rev-parse HEAD)" == "$(git rev-parse "origin/${BASE_BRANCH}")" ]]; then
-              echo "::notice::$BASE_BRANCH is up to date with $HEAD_BRANCH"
-              exit 0
-            fi
-            git push origin "$BASE_BRANCH"
-          elif [[ "$BASE_BRANCH" == "$SYNC_BRANCH" ]]; then
-            echo "::notice::Cannot fast-forward $BASE_BRANCH to $HEAD_BRANCH; merge existing PR first"
-          else
-            echo "continue=true" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Attempt merge into master
-        id: merge
-        if: steps.ff.outputs.continue
-        run: |
-          git switch master
-          if git merge "origin/${HEAD_BRANCH}"; then
-            if git diff HEAD^ --exit-code --stat; then
-              echo "::notice::No diff between master and $HEAD_BRANCH"
-              exit 0
-            fi
-          else
-            git merge --abort
-          fi
-          echo "continue=true" >> $GITHUB_OUTPUT
-
-      - name: Push sync branch
-        id: push
-        if: steps.merge.outputs.continue
-        run: |
-          git switch -c "$SYNC_BRANCH" "origin/${HEAD_BRANCH}"
-          git push origin "$SYNC_BRANCH"
-          echo "continue=true" >> $GITHUB_OUTPUT
-
-      - name: Open PR
-        if: steps.push.outputs.continue
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BODY: |
-            Please do a **normal merge**, not squash merge.
-            Please fix conflicts if necessary.
-
-            ---
-
-            Triggered by `${{ github.event_name }}`
-        run: |
-          url="$(
-            gh pr create --head "$SYNC_BRANCH" \
-            --title "🤖 Merge ${SYNC_BRANCH} into master" \
-            --body "$BODY"
-          )"
-          echo "::notice::PR created at ${url}"
-
-          gh pr edit "$SYNC_BRANCH" --add-label "sync" || true
+    if: github.repository_owner == 'cylc' || github.event_name != 'schedule'
+    uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1
+    with:
+      head_branch: ${{ inputs.head_branch }}
+    secrets: inherit

.github/workflows/build.yml

@@ -16,18 +16,21 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 10
     strategy:
+      fail-fast: false
       matrix:
         os: ['ubuntu-latest']
-        python: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11']
         include:
-          - os: 'macos-latest'
+          - os: 'ubuntu-22.04'
             python: '3.7'
+          - os: 'macos-latest'
+            python: '3.8'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 

.github/workflows/shortlog.yml

@@ -14,9 +14,10 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0  # need to fetch all commits to check contributors
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check CONTRIBUTING.md
         uses: cylc/release-actions/check-shortlog@v1

.github/workflows/test_conda-build.yml

@@ -15,38 +15,49 @@ concurrency:
 
 jobs:
   test_conda_install:
+    if: github.repository_owner == 'cylc' || github.event_name != 'schedule'
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3']
+    env:
+      ENV_FILE: conda-environment.yml
     steps:
       - name: checkout cylc-flow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: build conda env
+      - name: modify conda env file
         run: |
           # write environment file
-          env_file='conda-environment.yml'
-          echo "  - pip"  >> "$env_file"  # list pip as a dependency
-          echo "  - pip:" >> "$env_file"  # add a pip section
-          echo "    - ."  >> "$env_file"  # install cylc-flow (pip install .)
-          cat "$env_file"
-          # install environment
-          conda env create \
-            -f "$env_file" \
-            --prefix cylc-dev
-          . /usr/share/miniconda/etc/profile.d/conda.sh
-          # check cylc-flow was installed correctly
-          conda run --prefix cylc-dev cylc version --long
+          echo "  - pip"  >> "$ENV_FILE"  # list pip as a dependency
+          echo "  - pip:" >> "$ENV_FILE"  # add a pip section
+          echo "    - ."  >> "$ENV_FILE"  # install cylc-flow (pip install .)
+          cat "$ENV_FILE"
+
+      - name: build conda env
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          environment-file: ${{ env.ENV_FILE }}
+
+      - name: check cylc installation
+        shell: bash -el {0}
+        run: |
+          conda run -n test cylc version --long
 
       - name: check for activate scripts
+        shell : bash -el {0}
         run: |
           # https://github.com/cylc/cylc-flow/issues/3704#issuecomment-897442365
           # locate all activate scripts
-          find ./cylc-dev/ -name "activate.d" | tee > activates.txt
+          find "$CONDA_PREFIX" -name "activate.d" | tee > activates.txt
           # ignore the conda activate script itself
-          sed -i '/cylc-dev\/etc\/conda\/activate.d/d' activates.txt
+          sed -i "\|$CONDA_PREFIX/etc/conda/activate.d|d" activates.txt
           # check to make sure no packages have contributed new activate scripts
           # (we rely on having a conda activate-less environment)
-          if [[ $(cat activates.txt | wc -l) -ne 0 ]]; then
+          if [[ $(wc -l < activates.txt) -ne 0 ]]; then
               echo '::error::Found activate scripts in installation.'
               cat activates.txt >&2
               exit 1