Merge pull request #275 from DiamondLightSource/update_deps

Cherry-pick changes from the skeleton

.github/actions/install_requirements/action.yml

@@ -7,6 +7,9 @@ inputs:
   install_options:
     description: Parameters to pass to pip install
     required: true
+  artifact_name:
+    description: A user friendly name to give the produced artifacts
+    required: true
   python_version:
     description: Python version to install
     default: "3.x"
@@ -16,7 +19,7 @@ runs:
 
   steps:
     - name: Setup python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python_version }}
 
@@ -32,13 +35,13 @@ runs:
         mkdir -p lockfiles
         pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }}
         # delete the self referencing line and make sure it isn't blank
-        sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }}
+        sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }}
       shell: bash
 
     - name: Upload lockfiles
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4.0.0
       with:
-        name: lockfiles
+        name: lockfiles-${{ inputs.python_version }}-${{ inputs.artifact_name }}-${{ github.sha }}
         path: lockfiles
 
     # This eliminates the class of problems where the requirements being given no
@@ -55,4 +58,3 @@ runs:
           fi
         fi
       shell: bash
-

.github/dependabot.yml

@@ -9,6 +9,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    groups:
+      github-artifacts:
+        patterns:
+          - actions/*-artifact
 
   - package-ecosystem: "pip"
     directory: "/"

.github/pages/index.html

@@ -8,4 +8,4 @@
     <link rel="canonical" href="main/index.html">
 </head>
 
-</html>
+</html>

.github/pages/make_switcher.py

@@ -64,7 +64,7 @@ def write_json(path: Path, repository: str, versions: str):
     ]
     text = json.dumps(struct, indent=2)
     print(f"JSON switcher:\n{text}")
-    path.write_text(text)
+    path.write_text(text, encoding="utf-8")
 
 
 def main(args=None):

.github/workflows/code.yml

@@ -3,12 +3,10 @@ name: Code CI
 on:
   push:
   pull_request:
-  schedule:
-    # Run weekly to check latest versions of dependencies
-    - cron: "0 8 * * WED"
 env:
   # The target python version, which must match the Dockerfile version
   CONTAINER_PYTHON: "3.11"
+  DIST_WHEEL_PATH: dist-${{ github.sha }}
 
 jobs:
   lint:
@@ -26,6 +24,7 @@ jobs:
           requirements_file: requirements-dev-3.x.txt
           install_options: -e .[dev]
           python_version: "3.11"
+          artifact_name: lint
 
       - name: Lint
         run: tox -e pre-commit,mypy
@@ -62,6 +61,7 @@ jobs:
           python_version: ${{ matrix.python }}
           requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt
           install_options: ${{ matrix.install }}
+          artifact_name: tests
 
       - name: List dependency tree
         run: pipdeptree
@@ -92,9 +92,9 @@ jobs:
           pipx run build
 
       - name: Upload sdist and wheel as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.0.0
         with:
-          name: dist
+          name: ${{ env.DIST_WHEEL_PATH }}
           path: dist
 
       - name: Check for packaging errors
@@ -106,6 +106,7 @@ jobs:
           python_version: ${{env.CONTAINER_PYTHON}}
           requirements_file: requirements.txt
           install_options: dist/*.whl
+          artifact_name: dist
 
       - name: Test module --version works using the installed wheel
         # If more than one module in src/ replace with module name to test
@@ -119,6 +120,9 @@ jobs:
       contents: read
       packages: write
 
+    env:
+      TEST_TAG: "testing"
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -127,10 +131,15 @@ jobs:
       - name: Generate image repo name
         run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV
 
+      - name: Set lockfile location in environment
+        run: |
+          echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV
+
       - name: Download wheel and lockfiles
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4.1.0
         with:
           path: artifacts/
+          pattern: "*dist*"
 
       - name: Log in to GitHub Docker Registry
         if: github.event_name != 'pull_request'
@@ -140,39 +149,63 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Docker meta
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and export to Docker local cache
+        uses: docker/build-push-action@v5
+        with:
+          # Note build-args, context, file, and target must all match between this
+          # step and the later build-push-action, otherwise the second build-push-action
+          # will attempt to build the image again
+          build-args: |
+            PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl
+          context: artifacts/
+          file: ./Dockerfile
+          target: runtime
+          load: true
+          tags: ${{ env.TEST_TAG }}
+          # If you have a long docker build (2+ minutes), uncomment the
+          # following to turn on caching. For short build times this
+          # makes it a little slower
+          #cache-from: type=gha
+          #cache-to: type=gha,mode=max
+
+      - name: Create tags for publishing image
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.IMAGE_REPOSITORY }}
           tags: |
             type=ref,event=tag
-            type=raw,value=latest
+            type=raw,value=latest, enable=${{ github.ref_type == 'tag' }}
+          # type=edge,branch=main
+          # Add line above to generate image for every commit to given branch,
+          # and uncomment the end of if clause in next step
 
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build runtime image
+      - name: Push cached image to container registry
+        if: github.ref_type == 'tag' # || github.ref_name == 'main'
         uses: docker/build-push-action@v5
+        # This does not build the image again, it will find the image in the
+        # Docker cache and publish it
         with:
+          # Note build-args, context, file, and target must all match between this
+          # step and the previous build-push-action, otherwise this step will
+          # attempt to build the image again
           build-args: |
-            PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl
-          push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
-          load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+            PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl
           context: artifacts/
           file: ./Dockerfile
-          # If you have a long docker build, uncomment the following to turn on caching
-          # For short build times this makes it a little slower
-          #cache-from: type=gha
-          #cache-to: type=gha,mode=max
+          target: runtime
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
 
   release:
     # upload to PyPI and make a release on every tag
     needs: [lint, dist, test]
-    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
+    if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
     runs-on: ubuntu-latest
     env:
       HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }}
@@ -182,11 +215,15 @@ jobs:
       url: https://pypi.org/p/dls-dodal
 
     steps:
-      - uses: actions/download-artifact@v3
+      - name: Download wheel and lockfiles
+        uses: actions/[email protected]
+        with:
+          path: artifacts/
+          pattern: "*dist*"
 
       - name: Fixup blank lockfiles
         # Github release artifacts can't be blank
-        run: for f in lockfiles/*; do [ -s $f ] || echo '# No requirements' >> $f; done
+        run: for f in ${{ env.DIST_LOCKFILE_PATH }}/*; do [ -s $f ] || echo '# No requirements' >> $f; done
 
       - name: Github Release
         # We pin to the SHA, not the tag, for security reasons.
@@ -195,8 +232,8 @@ jobs:
         with:
           prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
           files: |
-            dist/*
-            lockfiles/*
+            ${{ env.DIST_WHEEL_PATH }}/*
+            ${{ env.DIST_LOCKFILE_PATH }}/*
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yml

@@ -30,6 +30,7 @@ jobs:
           requirements_file: requirements-dev-3.x.txt
           install_options: -e .[dev]
           python_version: "3.11"
+          artifact_name: docs
 
       - name: Build docs
         run: tox -e docs

.github/workflows/linkcheck.yml

@@ -20,6 +20,10 @@ jobs:
           requirements_file: requirements-dev-3.x.txt
           install_options: -e .[dev]
           python_version: "3.11"
+          artifact_name: link_check
 
       - name: Check links
         run: tox -e docs build -- -b linkcheck
+
+      - name: Keepalive Workflow
+        uses: gautamkrishnar/keepalive-workflow@v1

.vscode/extensions.json

@@ -7,4 +7,4 @@
         "ryanluker.vscode-coverage-gutters",
         "charliermarsh.Ruff"
     ]
-}
+}

.vscode/launch.json

@@ -23,4 +23,4 @@
             },
         }
     ]
-}
+}

.vscode/tasks.json

@@ -21,4 +21,4 @@
             "type": "shell"
         }
     ]
-}
+}

docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst

@@ -0,0 +1,35 @@
+2. Adopt python3-pip-skeleton for project structure
+===================================================
+
+Date: 2022-02-18
+
+Status
+------
+
+Accepted
+
+Context
+-------
+
+We should use the following `pip-skeleton <https://github.com/DiamondLightSource/python3-pip-skeleton>`_.
+The skeleton will ensure consistency in developer
+environments and package management.
+
+Decision
+--------
+
+We have switched to using the skeleton.
+
+Consequences
+------------
+
+This module will use a fixed set of tools as developed in python3-pip-skeleton
+and can pull from this skeleton to update the packaging to the latest techniques.
+
+As such, the developer environment may have changed, the following could be
+different:
+
+- linting
+- formatting
+- pip venv setup
+- CI/CD

docs/developer/how-to/build-docs.rst

@@ -13,7 +13,7 @@ docs that pull in docstrings from the code.
     `documentation_standards`
 
 The docs will be built into the ``build/html`` directory, and can be opened
-locally with a web browse::
+locally with a web browser::
 
     $ firefox build/html/index.html
 
@@ -35,4 +35,4 @@ changes in this directory too::
 
     $ tox -e docs autobuild -- --watch src
 
-.. _sphinx: https://www.sphinx-doc.org/
+.. _sphinx: https://www.sphinx-doc.org/

docs/developer/how-to/lint.rst

@@ -15,6 +15,9 @@ commit`` on just the files that have changed::
 
     $ pre-commit install
 
+It is also possible to `automatically enable pre-commit on cloned repositories <https://pre-commit.com/#automatically-enabling-pre-commit-on-repositories>`_.
+This will result in pre-commits being enabled on every repo your user clones from now on.
+
 Fixing issues
 -------------
 

docs/developer/how-to/make-release.rst

@@ -13,4 +13,4 @@ To make a new release, please follow this checklist:
 Note that tagging and pushing to the main branch has the same effect except that
 you will not get the option to edit the release notes.
 
-.. _release: https://github.com/DiamondLightSource/dodal/releases
+.. _release: https://github.com/DiamondLightSource/dodal/releases

docs/developer/reference/standards.rst

@@ -60,4 +60,4 @@ Docs follow the underlining convention::
 
 .. seealso::
 
-    How-to guide `../how-to/build-docs`
+    How-to guide `../how-to/build-docs`

docs/developer/tutorials/dev-install.rst

@@ -34,10 +34,15 @@ requires python 3.8 or later) or to run in a container under `VSCode
 
         .. code::
 
-            $ vscode dodal
+            $ code dodal
             # Click on 'Reopen in Container' when prompted
             # Open a new terminal
 
+        .. note::
+
+            See the epics-containers_ documentation for more complex
+            use cases, such as integration with podman.
+
 See what was installed
 ----------------------
 
@@ -58,3 +63,6 @@ This will run in parallel the following checks:
 - `../how-to/run-tests`
 - `../how-to/static-analysis`
 - `../how-to/lint`
+
+
+.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html