Merge branch 'master' into 8.2.x-sync

.github/workflows/1_create_release_pr.yml

@@ -21,7 +21,7 @@ jobs:
         uses: cylc/release-actions/stage-1/sanitize-inputs@v1
 
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.BASE_REF }}
           fetch-depth: 0  # need to fetch all commits to check contributors

.github/workflows/2_auto_publish_release.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
 
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ env.MERGE_SHA }}
 
@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/[email protected].8
+      uses: pypa/[email protected].10
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/bash.yml

@@ -44,7 +44,7 @@ jobs:
           - '5.0'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Docker container
         run: |

.github/workflows/build.yml

@@ -24,7 +24,7 @@ jobs:
             python: '3.7'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/shortlog.yml

@@ -14,7 +14,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0  # need to fetch all commits to check contributors
 

.github/workflows/test_conda-build.yml

@@ -19,7 +19,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: checkout cylc-flow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: build conda env
         run: |

.github/workflows/test_fast.yml

@@ -17,18 +17,18 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
-      fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs
+      fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3']
         include:
           - os: 'macos-latest'
             python-version: '3.7'
     env:
       PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
         uses: actions/setup-python@v4
@@ -39,7 +39,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           sudo apt-get update
-          sudo apt-get install -y shellcheck sqlite3
+          sudo apt-get install -y sqlite3
 
       - name: Install
         run: |
@@ -48,37 +48,10 @@ jobs:
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
-      - name: Check changelog
-        if: startsWith(matrix.os, 'ubuntu')
-        run: towncrier build --draft
-
-      - name: Style
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          flake8
-          etc/bin/shellchecker
-
-      # note: exclude python 3.10+ from mypy checks as these produce false
-      # positives in installed libraries for python 3.7
-      - name: Typing
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python-version, 3.9)
-        run: mypy
-
-      - name: Doctests
-        timeout-minutes: 4
-        run: |
-          pytest cylc/flow
-
       - name: Unit Tests
-        timeout-minutes: 4
+        timeout-minutes: 5
         run: |
-          pytest tests/unit
-
-      - name: Bandit
-        if: ${{ matrix.python-version == '3.7' }}
-        # https://github.com/PyCQA/bandit/issues/658
-        run: |
-          bandit -r --ini .bandit cylc/flow
+          pytest cylc/flow tests/unit
 
       - name: Integration Tests
         timeout-minutes: 6
@@ -104,17 +77,55 @@ jobs:
           path: coverage.xml
           retention-days: 7
 
+  lint:
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 10
+    steps:
+      - name: Apt-Get Install
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # note: exclude python 3.10+ from mypy checks as these produce false
+      # positives in installed libraries for python 3.7
+      - name: Configure Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install
+        run: |
+          pip install -e ."[tests]"
+
+      - name: Flake8
+        run: flake8
+
+      - name: Bandit
+        run: |
+          bandit -r --ini .bandit cylc/flow
+
+      - name: Shellchecker
+        run: etc/bin/shellchecker
+
+      - name: MyPy
+        run: mypy
+
+      - name: Towncrier
+        run: towncrier build --draft
+
       - name: Linkcheck
-        if: startsWith(matrix.python-version, '3.10')
-        run: pytest -m linkcheck --dist=load tests/unit
+        run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py
 
   codecov:
     needs: test
     runs-on: ubuntu-latest
     timeout-minutes: 2
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download coverage artifacts
         uses: actions/download-artifact@v3

.github/workflows/test_functional.yml

@@ -46,9 +46,9 @@ jobs:
         # NOTE: includes must define ALL of the matrix values
         include:
           # latest python
-          - name: 'py-3.11'
+          - name: 'py-3-latest'
             os: 'ubuntu-latest'
-            python-version: '3.11'
+            python-version: '3'
             test-base: 'tests/f'
             chunk: '1/4'
             platform: '_local_background*'
@@ -96,7 +96,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
         uses: actions/setup-python@v4
@@ -108,7 +108,7 @@ jobs:
         run: |
           # install system deps
           brew update
-          brew install bash coreutils gnu-sed
+          brew install bash coreutils gnu-sed grep
 
           # add GNU coreutils and sed to the user PATH
           # (see instructions in brew install output)
@@ -118,6 +118,9 @@ jobs:
           echo \
             "/usr/local/opt/gnu-sed/libexec/gnubin" \
             >> "${GITHUB_PATH}"
+          echo \
+            "/usr/local/opt/grep/libexec/gnubin" \
+            >> "${GITHUB_PATH}"
 
           # add coreutils to the bashrc too (for jobs)
           cat >> "${HOME}/.bashrc" <<__HERE__
@@ -306,7 +309,7 @@ jobs:
     timeout-minutes: 2
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Download coverage artifacts
         uses: actions/download-artifact@v3

.github/workflows/test_manylinux.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1

.github/workflows/test_tutorial_workflow.yml

@@ -31,7 +31,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install docs/tutorial dependencies
         uses: cylc/cylc-doc/.github/actions/install-dependencies@master

.github/workflows/update_copyright.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
 
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Configure git
       uses: cylc/release-actions/configure-git@v1

CONTRIBUTING.md

@@ -64,7 +64,7 @@ requests_).
  - Prasanna Challuri
  - David Matthews
  - Tim Whitcomb
- - (Scott Wales)
+ - Scott Wales
  - Tomek Trzeciak
  - Thomas Coleman
  - Bruno Kinoshita
@@ -92,6 +92,7 @@ requests_).
  - John Haiducek
  - (Andrew Huang)
  - Cheng Da
+ - Mark Dawson
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version

changes.d/5571.feat.md

@@ -0,0 +1 @@
+Make workflow `CYLC_` variables available to the template processor during parsing.

changes.d/5709.feat.md

@@ -0,0 +1 @@
+Forward arbitrary environment variables over SSH connections

changes.d/5727.break.md

@@ -0,0 +1 @@
+Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value. If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`.

changes.d/5794.break.md

@@ -0,0 +1 @@
+Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12.

conda-environment.yml

@@ -10,13 +10,13 @@ dependencies:
   # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat
   - jinja2 >=3.0,<3.1
   - metomi-isodatetime >=1!3.0.0, <1!3.2.0
+  - packaging
   # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
   - protobuf >=4.21.2,<4.22.0
   - psutil >=5.6.0
   - python
   - pyzmq >=22
-  - setuptools >=49,!=67.*
-  - importlib_metadata # [py<3.8]
+  - importlib_metadata >=5.0 # [py<3.12]
   - urwid >=2,<3
   - tomli >=2 # [py<3.11]
 

cylc/flow/__init__.py

@@ -53,16 +53,24 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.2.4.dev'
+__version__ = '8.3.0.dev'
 
 
 def iter_entry_points(entry_point_name):
     """Iterate over Cylc entry points."""
-    import pkg_resources
+    import sys
+    if sys.version_info[:2] > (3, 11):
+        from importlib.metadata import entry_points
+    else:
+        # BACK COMPAT: importlib_metadata
+        #   importlib.metadata was added in Python 3.8. The required interfaces
+        #   were completed by 3.12. For lower versions we must use the
+        #   importlib_metadata backport.
+        # FROM: Python 3.7
+        # TO: Python: 3.12
+        from importlib_metadata import entry_points
     yield from (
         entry_point
-        for entry_point in pkg_resources.iter_entry_points(entry_point_name)
-        # Filter out the cylc namespace as it should be empty.
-        # All cylc packages should take the form cylc-<name>
-        if entry_point.dist.key != 'cylc'
+        # for entry_point in entry_points()[entry_point_name]
+        for entry_point in entry_points().select(group=entry_point_name)
     )