From 8bcaad2418d4ca61409e496fde16d770af08911d Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Fri, 27 Jan 2023 11:07:39 +0000 Subject: [PATCH 01/24] Removed scheduled job in code.yaml and added keepalive-workflow --- .github/workflows/code.yml | 3 --- .github/workflows/linkcheck.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 5495e291cd..201614aeb4 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -3,9 +3,6 @@ name: Code CI on: push: pull_request: - schedule: - # Run weekly to check latest versions of dependencies - - cron: "0 8 * * WED" env: # The target python version, which must match the Dockerfile version CONTAINER_PYTHON: "3.11" diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index e2d2be053a..37f554a7e9 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -23,3 +23,6 @@ jobs: - name: Check links run: tox -e docs build -- -b linkcheck + + - name: Keepalive Workflow + uses: gautamkrishnar/keepalive-workflow@v1 \ No newline at end of file From 4eafb4a7bbe15699443a79af7d90fe16e785d8da Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Mon, 6 Feb 2023 11:54:01 +0000 Subject: [PATCH 02/24] Added the link to autogenerate precommit --- docs/developer/how-to/lint.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/developer/how-to/lint.rst b/docs/developer/how-to/lint.rst index abbda365ac..2df258d8fc 100644 --- a/docs/developer/how-to/lint.rst +++ b/docs/developer/how-to/lint.rst @@ -15,6 +15,9 @@ commit`` on just the files that have changed:: $ pre-commit install +It is also possible to `automatically enable pre-commit on cloned repositories `_. +This will result in pre-commits being enabled on every repo your user clones from now on. + Fixing issues ------------- From 026cc3fdb45701325ae7070971b1191003f98d7f Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Fri, 20 Jan 2023 13:57:36 +0000 Subject: [PATCH 03/24] Made suggested changes --- .../0002-switched-to-pip-skeleton.rst | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst diff --git a/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst b/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst new file mode 100644 index 0000000000..41d90fd4ac --- /dev/null +++ b/docs/developer/explanations/decisions/0002-switched-to-pip-skeleton.rst @@ -0,0 +1,35 @@ +2. Adopt python3-pip-skeleton for project structure +=================================================== + +Date: 2022-02-18 + +Status +------ + +Accepted + +Context +------- + +We should use the following `pip-skeleton `_. +The skeleton will ensure consistency in developer +environments and package management. + +Decision +-------- + +We have switched to using the skeleton. + +Consequences +------------ + +This module will use a fixed set of tools as developed in python3-pip-skeleton +and can pull from this skeleton to update the packaging to the latest techniques. + +As such, the developer environment may have changed, the following could be +different: + +- linting +- formatting +- pip venv setup +- CI/CD From f8b75593841e374e67b4c639701696df86dc4f76 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Thu, 2 Mar 2023 13:16:56 +0000 Subject: [PATCH 04/24] Add explicit dependency on pytest --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8517ee01e0..b0f3c989dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dev = [ "pipdeptree", "pre-commit", "pydata-sphinx-theme>=0.12", + "pytest", "pytest-asyncio", "pytest-cov", "pytest-random-order", From e5cb2293b6cd3c6ca73b6da62d922b4fd92d083e Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Fri, 24 Mar 2023 14:41:47 +0000 Subject: [PATCH 05/24] Made changes --- docs/developer/how-to/build-docs.rst | 2 +- src/dodal/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developer/how-to/build-docs.rst b/docs/developer/how-to/build-docs.rst index 79e3f780cb..0174fc82dd 100644 --- a/docs/developer/how-to/build-docs.rst +++ b/docs/developer/how-to/build-docs.rst @@ -13,7 +13,7 @@ docs that pull in docstrings from the code. `documentation_standards` The docs will be built into the ``build/html`` directory, and can be opened -locally with a web browse:: +locally with a web browser:: $ firefox build/html/index.html diff --git a/src/dodal/__main__.py b/src/dodal/__main__.py index 2b8becbc9f..47ff5542e3 100644 --- a/src/dodal/__main__.py +++ b/src/dodal/__main__.py @@ -7,7 +7,7 @@ def main(args=None): parser = ArgumentParser() - parser.add_argument("--version", action="version", version=__version__) + parser.add_argument("-v", "--version", action="version", version=__version__) args = parser.parse_args(args) From 0029db8183c8689bbce86e8b30c7a1e109a1f44c Mon Sep 17 00:00:00 2001 From: Callum Forrester Date: Tue, 18 Apr 2023 16:16:32 +0100 Subject: [PATCH 06/24] Include link in dev install tutorial to epics-containers devcontainer page --- docs/developer/tutorials/dev-install.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/developer/tutorials/dev-install.rst b/docs/developer/tutorials/dev-install.rst index c7f2a79a0f..936369327c 100644 --- a/docs/developer/tutorials/dev-install.rst +++ b/docs/developer/tutorials/dev-install.rst @@ -34,9 +34,14 @@ requires python 3.8 or later) or to run in a container under `VSCode .. code:: - $ vscode dodal + $ code dodal # Click on 'Reopen in Container' when prompted # Open a new terminal + + .. note:: + + See the epics-containers_ documentation for more complex + use cases, such as integration with podman. See what was installed ---------------------- @@ -58,3 +63,6 @@ This will run in parallel the following checks: - `../how-to/run-tests` - `../how-to/static-analysis` - `../how-to/lint` + + +.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html \ No newline at end of file From 973e09acd7bfbc8df635da50de296158ccbb9425 Mon Sep 17 00:00:00 2001 From: AlexWells Date: Tue, 30 May 2023 12:23:29 +0100 Subject: [PATCH 07/24] Improve container build workflow This will test the container before it is pushed to GHCR --- .github/workflows/code.yml | 60 +++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 201614aeb4..4f717dbc86 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -116,6 +116,9 @@ jobs: contents: read packages: write + env: + TEST_TAG: "testing" + steps: - name: Checkout uses: actions/checkout@v4 @@ -137,39 +140,70 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and export to Docker local cache + uses: docker/build-push-action@v4 + with: + # Note build-args, context, file, and target must all match between this + # step and the later build-push-action, otherwise the second build-push-action + # will attempt to build the image again + build-args: | + PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl + context: artifacts/ + file: ./Dockerfile + target: runtime + load: true + tags: ${{ env.TEST_TAG }} + # If you have a long docker build (2+ minutes), uncomment the + # following to turn on caching. For short build times this + # makes it a little slower + #cache-from: type=gha + #cache-to: type=gha,mode=max + + - name: Test cli works in cached runtime image + run: docker run docker.io/library/${{ env.TEST_TAG }} --version + + - name: Create tags for publishing image id: meta uses: docker/metadata-action@v5 with: images: ${{ env.IMAGE_REPOSITORY }} tags: | type=ref,event=tag - type=raw,value=latest + type=raw,value=latest, enable=${{ github.ref_type == 'tag' }} + # type=edge,branch=main + # Add line above to generate image for every commit to given branch, + # and uncomment the end of if clause in next step - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - - name: Build runtime image - uses: docker/build-push-action@v5 + - name: Push cached image to container registry + if: github.ref_type == 'tag' # || github.ref_name == 'main' + uses: docker/build-push-action@v3 + # This does not build the image again, it will find the image in the + # Docker cache and publish it with: + # Note build-args, context, file, and target must all match between this + # step and the previous build-push-action, otherwise this step will + # attempt to build the image again build-args: | PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl - push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} - load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} context: artifacts/ file: ./Dockerfile - # If you have a long docker build, uncomment the following to turn on caching - # For short build times this makes it a little slower - #cache-from: type=gha - #cache-to: type=gha,mode=max + target: runtime + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} release: # upload to PyPI and make a release on every tag needs: [lint, dist, test] - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }} + if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} runs-on: ubuntu-latest env: HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }} From 3b51e13826c9ecd7f3d53e69a514d9d85023efd6 Mon Sep 17 00:00:00 2001 From: Eva Lott Date: Tue, 20 Jun 2023 11:29:18 +0100 Subject: [PATCH 08/24] Slight changes to the sed command, and ensuring output is utf-8 on windows --- .github/actions/install_requirements/action.yml | 2 +- .github/pages/make_switcher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 25a146d164..c5d4db43b7 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -32,7 +32,7 @@ runs: mkdir -p lockfiles pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }} # delete the self referencing line and make sure it isn't blank - sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }} + sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }} shell: bash - name: Upload lockfiles diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py index 2d1915f1c7..ae227ab7fd 100755 --- a/.github/pages/make_switcher.py +++ b/.github/pages/make_switcher.py @@ -64,7 +64,7 @@ def write_json(path: Path, repository: str, versions: str): ] text = json.dumps(struct, indent=2) print(f"JSON switcher:\n{text}") - path.write_text(text) + path.write_text(text, encoding="utf-8") def main(args=None): From 35b96485ac9e8e20579788e364857cef0d4d0fca Mon Sep 17 00:00:00 2001 From: AlexWells Date: Fri, 1 Sep 2023 13:47:50 +0100 Subject: [PATCH 09/24] Remove trailing spaces and rationalise newlines Some tools remove trailing whitespace by default on save, so may as well correct the originals. Ensures every file ends with an empty blank line. Again some tools do this automatically. --- .github/actions/install_requirements/action.yml | 1 - .github/pages/index.html | 2 +- .github/workflows/linkcheck.yml | 2 +- .vscode/extensions.json | 2 +- .vscode/launch.json | 2 +- .vscode/tasks.json | 2 +- docs/developer/how-to/build-docs.rst | 2 +- docs/developer/how-to/make-release.rst | 2 +- docs/developer/reference/standards.rst | 2 +- docs/developer/tutorials/dev-install.rst | 10 +++++----- 10 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index c5d4db43b7..20d7a3adf1 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -55,4 +55,3 @@ runs: fi fi shell: bash - diff --git a/.github/pages/index.html b/.github/pages/index.html index 80f0a00912..c495f39f2f 100644 --- a/.github/pages/index.html +++ b/.github/pages/index.html @@ -8,4 +8,4 @@ - \ No newline at end of file + diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 37f554a7e9..ba5d478c3c 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -25,4 +25,4 @@ jobs: run: tox -e docs build -- -b linkcheck - name: Keepalive Workflow - uses: gautamkrishnar/keepalive-workflow@v1 \ No newline at end of file + uses: gautamkrishnar/keepalive-workflow@v1 diff --git a/.vscode/extensions.json b/.vscode/extensions.json index e3b582fde2..a1227b3482 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -7,4 +7,4 @@ "ryanluker.vscode-coverage-gutters", "charliermarsh.Ruff" ] -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index a5fe2336c3..a14b412778 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -23,4 +23,4 @@ }, } ] -} \ No newline at end of file +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 1b21c0fb27..4ac17c12eb 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -21,4 +21,4 @@ "type": "shell" } ] -} \ No newline at end of file +} diff --git a/docs/developer/how-to/build-docs.rst b/docs/developer/how-to/build-docs.rst index 0174fc82dd..11a5e6386d 100644 --- a/docs/developer/how-to/build-docs.rst +++ b/docs/developer/how-to/build-docs.rst @@ -35,4 +35,4 @@ changes in this directory too:: $ tox -e docs autobuild -- --watch src -.. _sphinx: https://www.sphinx-doc.org/ \ No newline at end of file +.. _sphinx: https://www.sphinx-doc.org/ diff --git a/docs/developer/how-to/make-release.rst b/docs/developer/how-to/make-release.rst index ad3654821c..a7f82ff8a1 100644 --- a/docs/developer/how-to/make-release.rst +++ b/docs/developer/how-to/make-release.rst @@ -13,4 +13,4 @@ To make a new release, please follow this checklist: Note that tagging and pushing to the main branch has the same effect except that you will not get the option to edit the release notes. -.. _release: https://github.com/DiamondLightSource/dodal/releases \ No newline at end of file +.. _release: https://github.com/DiamondLightSource/dodal/releases diff --git a/docs/developer/reference/standards.rst b/docs/developer/reference/standards.rst index 7462b4bac9..5a1fd47825 100644 --- a/docs/developer/reference/standards.rst +++ b/docs/developer/reference/standards.rst @@ -60,4 +60,4 @@ Docs follow the underlining convention:: .. seealso:: - How-to guide `../how-to/build-docs` \ No newline at end of file + How-to guide `../how-to/build-docs` diff --git a/docs/developer/tutorials/dev-install.rst b/docs/developer/tutorials/dev-install.rst index 936369327c..d0d3c56774 100644 --- a/docs/developer/tutorials/dev-install.rst +++ b/docs/developer/tutorials/dev-install.rst @@ -37,10 +37,10 @@ requires python 3.8 or later) or to run in a container under `VSCode $ code dodal # Click on 'Reopen in Container' when prompted # Open a new terminal - - .. note:: - - See the epics-containers_ documentation for more complex + + .. note:: + + See the epics-containers_ documentation for more complex use cases, such as integration with podman. See what was installed @@ -65,4 +65,4 @@ This will run in parallel the following checks: - `../how-to/lint` -.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html \ No newline at end of file +.. _epics-containers: https://epics-containers.github.io/main/user/tutorials/devcontainer.html From 1665e223dd0ec5123107eef2d0f9b87e928e3ba1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:54:19 +0000 Subject: [PATCH 10/24] Bump docker/build-push-action from 3 to 5 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 3 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v3...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/code.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 4f717dbc86..08605f842d 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -145,7 +145,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Build and export to Docker local cache - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: # Note build-args, context, file, and target must all match between this # step and the later build-push-action, otherwise the second build-push-action @@ -184,7 +184,7 @@ jobs: - name: Push cached image to container registry if: github.ref_type == 'tag' # || github.ref_name == 'main' - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 # This does not build the image again, it will find the image in the # Docker cache and publish it with: From a25a975a0a8b5bd3e09c5ef8e17312bf10e78783 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 16:31:41 +0000 Subject: [PATCH 11/24] Bump docker/setup-buildx-action from 2 to 3 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 08605f842d..dfbad03163 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -142,7 +142,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build and export to Docker local cache uses: docker/build-push-action@v5 From 9fd3168f97e96173c6c9454e43cb05044bd49aee Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 12:00:53 +0000 Subject: [PATCH 12/24] Bump upload/download artifact and configure dependabot to group them --- .github/actions/install_requirements/action.yml | 2 +- .github/dependabot.yml | 4 ++++ .github/workflows/code.yml | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 20d7a3adf1..10b985430b 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -36,7 +36,7 @@ runs: shell: bash - name: Upload lockfiles - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: lockfiles path: lockfiles diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fb7c6ee671..2d1af8738d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,10 @@ updates: directory: "/" schedule: interval: "weekly" + groups: + github-artifacts: + patterns: + - actions/*-artifact - package-ecosystem: "pip" directory: "/" diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index dfbad03163..d3738e87e3 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -89,7 +89,7 @@ jobs: pipx run build - name: Upload sdist and wheel as artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist @@ -128,7 +128,7 @@ jobs: run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV - name: Download wheel and lockfiles - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts/ @@ -213,7 +213,7 @@ jobs: url: https://pypi.org/p/dls-dodal steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 - name: Fixup blank lockfiles # Github release artifacts can't be blank From 1eee9d556ec1c836b00b6f16e786404ba7425b28 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 13:03:07 +0000 Subject: [PATCH 13/24] Upload artifacts with unique names --- .github/actions/install_requirements/action.yml | 2 +- .github/workflows/code.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 10b985430b..b0d6802717 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -38,7 +38,7 @@ runs: - name: Upload lockfiles uses: actions/upload-artifact@v4 with: - name: lockfiles + name: lockfiles-${{ inputs.python_version }}-${{ github.sha }} path: lockfiles # This eliminates the class of problems where the requirements being given no diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index d3738e87e3..ca7441373a 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -91,7 +91,7 @@ jobs: - name: Upload sdist and wheel as artifacts uses: actions/upload-artifact@v4 with: - name: dist + name: dist-${{ github.sha }} path: dist - name: Check for packaging errors From e8dff7dc61eaa6aad10cb12d898059fbb71cd395 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 13:13:06 +0000 Subject: [PATCH 14/24] Specify different artifact names each time install requirements is run --- .github/actions/install_requirements/action.yml | 5 ++++- .github/workflows/code.yml | 2 ++ .github/workflows/docs.yml | 1 + .github/workflows/linkcheck.yml | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index b0d6802717..0edab8ad30 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -10,6 +10,9 @@ inputs: python_version: description: Python version to install default: "3.x" + artifact_name: + description: A user friendly name to give the produced artifacts + default: "tests" runs: using: composite @@ -38,7 +41,7 @@ runs: - name: Upload lockfiles uses: actions/upload-artifact@v4 with: - name: lockfiles-${{ inputs.python_version }}-${{ github.sha }} + name: lockfiles-${{ inputs.python_version }}-${{ inputs.artifact_name }}-${{ github.sha }} path: lockfiles # This eliminates the class of problems where the requirements being given no diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index ca7441373a..00441cee80 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -23,6 +23,7 @@ jobs: requirements_file: requirements-dev-3.x.txt install_options: -e .[dev] python_version: "3.11" + artifact_name: lint - name: Lint run: tox -e pre-commit,mypy @@ -103,6 +104,7 @@ jobs: python_version: ${{env.CONTAINER_PYTHON}} requirements_file: requirements.txt install_options: dist/*.whl + artifact_name: dist - name: Test module --version works using the installed wheel # If more than one module in src/ replace with module name to test diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 821bc7fce1..d6e4b0e5f4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,6 +30,7 @@ jobs: requirements_file: requirements-dev-3.x.txt install_options: -e .[dev] python_version: "3.11" + artifact_name: docs - name: Build docs run: tox -e docs diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index ba5d478c3c..ed9cf57f3e 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -20,6 +20,7 @@ jobs: requirements_file: requirements-dev-3.x.txt install_options: -e .[dev] python_version: "3.11" + artifact_name: link_check - name: Check links run: tox -e docs build -- -b linkcheck From 30f16cf63ea1513e039144b696f2405453b6345c Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 13:21:49 +0000 Subject: [PATCH 15/24] Point downloaded artifacts to new names --- .github/workflows/code.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 00441cee80..9690c2db02 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -153,7 +153,7 @@ jobs: # step and the later build-push-action, otherwise the second build-push-action # will attempt to build the image again build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl + PIP_OPTIONS=-r lockfiles/requirements.txt dist-${{ github.sha }}/*.whl context: artifacts/ file: ./Dockerfile target: runtime @@ -194,7 +194,7 @@ jobs: # step and the previous build-push-action, otherwise this step will # attempt to build the image again build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl + PIP_OPTIONS=-r lockfiles/requirements.txt dist-${{ github.sha }}/*.whl context: artifacts/ file: ./Dockerfile target: runtime @@ -219,7 +219,7 @@ jobs: - name: Fixup blank lockfiles # Github release artifacts can't be blank - run: for f in lockfiles/*; do [ -s $f ] || echo '# No requirements' >> $f; done + run: for f in lockfiles*dist*/*; do [ -s $f ] || echo '# No requirements' >> $f; done - name: Github Release # We pin to the SHA, not the tag, for security reasons. @@ -228,8 +228,8 @@ jobs: with: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} files: | - dist/* - lockfiles/* + dist-${{ github.sha }}/* + lockfiles*dist*/* generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d9af54ee2355943727e4de549f3dca81a3d228e5 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 13:30:44 +0000 Subject: [PATCH 16/24] Pull dist lockfile location into variable --- .github/workflows/code.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 9690c2db02..a3d7d97993 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -6,6 +6,7 @@ on: env: # The target python version, which must match the Dockerfile version CONTAINER_PYTHON: "3.11" + DIST_LOCKFILE_PATH: lockfiles-${{env.CONTAINER_PYTHON}}-dist-${{ github.sha }} jobs: lint: @@ -153,7 +154,7 @@ jobs: # step and the later build-push-action, otherwise the second build-push-action # will attempt to build the image again build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist-${{ github.sha }}/*.whl + PIP_OPTIONS=-r ${{env.DIST_LOCKFILE_PATH}}/requirements.txt dist-${{ github.sha }}/*.whl context: artifacts/ file: ./Dockerfile target: runtime @@ -194,7 +195,7 @@ jobs: # step and the previous build-push-action, otherwise this step will # attempt to build the image again build-args: | - PIP_OPTIONS=-r lockfiles/requirements.txt dist-${{ github.sha }}/*.whl + PIP_OPTIONS=-r ${{env.DIST_LOCKFILE_PATH}}/requirements.txt dist-${{ github.sha }}/*.whl context: artifacts/ file: ./Dockerfile target: runtime @@ -219,7 +220,7 @@ jobs: - name: Fixup blank lockfiles # Github release artifacts can't be blank - run: for f in lockfiles*dist*/*; do [ -s $f ] || echo '# No requirements' >> $f; done + run: for f in ${{env.DIST_LOCKFILE_PATH}}/*; do [ -s $f ] || echo '# No requirements' >> $f; done - name: Github Release # We pin to the SHA, not the tag, for security reasons. @@ -229,7 +230,7 @@ jobs: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} files: | dist-${{ github.sha }}/* - lockfiles*dist*/* + ${{env.DIST_LOCKFILE_PATH}}/* generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 71671138661779bb7ac50b7670be2aedc3e3ee95 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 13:46:03 +0000 Subject: [PATCH 17/24] Set lockfile/wheel locations into env manually --- .github/workflows/code.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index a3d7d97993..39f5bf4221 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -6,7 +6,6 @@ on: env: # The target python version, which must match the Dockerfile version CONTAINER_PYTHON: "3.11" - DIST_LOCKFILE_PATH: lockfiles-${{env.CONTAINER_PYTHON}}-dist-${{ github.sha }} jobs: lint: @@ -85,6 +84,11 @@ jobs: # Need this to get version number from last tag fetch-depth: 0 + - name: Set artifact locations in environment + run: | + echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV + echo "DIST_WHEEL_PATH=dist-${{ github.sha }}" >> $GITHUB_ENV + - name: Build sdist and wheel run: | export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ @@ -93,7 +97,7 @@ jobs: - name: Upload sdist and wheel as artifacts uses: actions/upload-artifact@v4 with: - name: dist-${{ github.sha }} + name: ${{ env.DIST_WHEEL_PATH }} path: dist - name: Check for packaging errors @@ -154,7 +158,7 @@ jobs: # step and the later build-push-action, otherwise the second build-push-action # will attempt to build the image again build-args: | - PIP_OPTIONS=-r ${{env.DIST_LOCKFILE_PATH}}/requirements.txt dist-${{ github.sha }}/*.whl + PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl context: artifacts/ file: ./Dockerfile target: runtime @@ -195,7 +199,7 @@ jobs: # step and the previous build-push-action, otherwise this step will # attempt to build the image again build-args: | - PIP_OPTIONS=-r ${{env.DIST_LOCKFILE_PATH}}/requirements.txt dist-${{ github.sha }}/*.whl + PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl context: artifacts/ file: ./Dockerfile target: runtime @@ -220,7 +224,7 @@ jobs: - name: Fixup blank lockfiles # Github release artifacts can't be blank - run: for f in ${{env.DIST_LOCKFILE_PATH}}/*; do [ -s $f ] || echo '# No requirements' >> $f; done + run: for f in ${{ env.DIST_LOCKFILE_PATH }}/*; do [ -s $f ] || echo '# No requirements' >> $f; done - name: Github Release # We pin to the SHA, not the tag, for security reasons. @@ -229,8 +233,8 @@ jobs: with: prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }} files: | - dist-${{ github.sha }}/* - ${{env.DIST_LOCKFILE_PATH}}/* + ${{ env.DIST_WHEEL_PATH }}/* + ${{ env.DIST_LOCKFILE_PATH }}/* generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From a4847e1303f97c8b52f3ef486df83e2c2be2c726 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 13:55:04 +0000 Subject: [PATCH 18/24] Move setting environment variables --- .github/workflows/code.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 39f5bf4221..80baf7480f 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -6,6 +6,7 @@ on: env: # The target python version, which must match the Dockerfile version CONTAINER_PYTHON: "3.11" + DIST_WHEEL_PATH: dist-${{ github.sha }} jobs: lint: @@ -84,11 +85,6 @@ jobs: # Need this to get version number from last tag fetch-depth: 0 - - name: Set artifact locations in environment - run: | - echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV - echo "DIST_WHEEL_PATH=dist-${{ github.sha }}" >> $GITHUB_ENV - - name: Build sdist and wheel run: | export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ @@ -134,6 +130,10 @@ jobs: - name: Generate image repo name run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV + - name: Set lockfile location in environment + run: | + echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV + - name: Download wheel and lockfiles uses: actions/download-artifact@v4 with: From a6b3377884b88bef5ee7fec498690ac195019022 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 14:11:49 +0000 Subject: [PATCH 19/24] Minor updates from review comments --- .github/actions/install_requirements/action.yml | 8 ++++---- .github/workflows/code.yml | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index 0edab8ad30..e67685cec8 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -7,12 +7,12 @@ inputs: install_options: description: Parameters to pass to pip install required: true + artifact_name: + description: A user friendly name to give the produced artifacts + required: true python_version: description: Python version to install default: "3.x" - artifact_name: - description: A user friendly name to give the produced artifacts - default: "tests" runs: using: composite @@ -39,7 +39,7 @@ runs: shell: bash - name: Upload lockfiles - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v4.0.0 with: name: lockfiles-${{ inputs.python_version }}-${{ inputs.artifact_name }}-${{ github.sha }} path: lockfiles diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 80baf7480f..0acbe6f01b 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -61,6 +61,7 @@ jobs: python_version: ${{ matrix.python }} requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt install_options: ${{ matrix.install }} + artifact_name: tests - name: List dependency tree run: pipdeptree @@ -91,7 +92,7 @@ jobs: pipx run build - name: Upload sdist and wheel as artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v4.0.0 with: name: ${{ env.DIST_WHEEL_PATH }} path: dist @@ -135,7 +136,7 @@ jobs: echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV - name: Download wheel and lockfiles - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v4.0.0 with: path: artifacts/ @@ -220,7 +221,7 @@ jobs: url: https://pypi.org/p/dls-dodal steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4.0.0 - name: Fixup blank lockfiles # Github release artifacts can't be blank From ca10ef2278eaad6bd97f82a69e9c35515d0c2044 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 13:59:47 +0000 Subject: [PATCH 20/24] Update setup_python to version 5 --- .github/actions/install_requirements/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml index e67685cec8..79d1a71eef 100644 --- a/.github/actions/install_requirements/action.yml +++ b/.github/actions/install_requirements/action.yml @@ -19,7 +19,7 @@ runs: steps: - name: Setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python_version }} From 8055d14fc38284175188e7259c60df54b7c5b6c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 14:16:53 +0000 Subject: [PATCH 21/24] Bump the github-artifacts group with 1 update Bumps the github-artifacts group with 1 update: [actions/download-artifact](https://github.com/actions/download-artifact). Updates `actions/download-artifact` from 4.0.0 to 4.1.0 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-artifacts ... Signed-off-by: dependabot[bot] --- .github/workflows/code.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 0acbe6f01b..417f76e5e5 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -136,7 +136,7 @@ jobs: echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV - name: Download wheel and lockfiles - uses: actions/download-artifact@v4.0.0 + uses: actions/download-artifact@v4.1.0 with: path: artifacts/ @@ -221,7 +221,7 @@ jobs: url: https://pypi.org/p/dls-dodal steps: - - uses: actions/download-artifact@v4.0.0 + - uses: actions/download-artifact@v4.1.0 - name: Fixup blank lockfiles # Github release artifacts can't be blank From a518b81e269dd7f34f5a9cbb38c36d8bc90c5cf8 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 15:52:01 +0000 Subject: [PATCH 22/24] Only download required dist artifacts --- .github/workflows/code.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 417f76e5e5..17f7a8524a 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -139,6 +139,7 @@ jobs: uses: actions/download-artifact@v4.1.0 with: path: artifacts/ + pattern: "*dist*" - name: Log in to GitHub Docker Registry if: github.event_name != 'pull_request' @@ -221,7 +222,11 @@ jobs: url: https://pypi.org/p/dls-dodal steps: - - uses: actions/download-artifact@v4.1.0 + - name: Download wheel and lockfiles + uses: actions/download-artifact@v4.1.0 + with: + path: artifacts/ + pattern: "*dist*" - name: Fixup blank lockfiles # Github release artifacts can't be blank From c9823a270cd8b11b0c7473f9e3eb79cad2aec23b Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 16:46:37 +0000 Subject: [PATCH 23/24] Fix merge mistake from skeleton --- .github/workflows/code.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index 17f7a8524a..fbb5f248e2 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -187,10 +187,6 @@ jobs: # Add line above to generate image for every commit to given branch, # and uncomment the end of if clause in next step - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - name: Push cached image to container registry if: github.ref_type == 'tag' # || github.ref_name == 'main' uses: docker/build-push-action@v5 From 56ce77183b81eb8940affbc5d14508db48b421e8 Mon Sep 17 00:00:00 2001 From: Dominic Oram Date: Thu, 4 Jan 2024 17:02:39 +0000 Subject: [PATCH 24/24] Do not run CLI as this doesn't make sense for dodal --- .github/workflows/code.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml index fbb5f248e2..bd5bdb1ad5 100644 --- a/.github/workflows/code.yml +++ b/.github/workflows/code.yml @@ -172,9 +172,6 @@ jobs: #cache-from: type=gha #cache-to: type=gha,mode=max - - name: Test cli works in cached runtime image - run: docker run docker.io/library/${{ env.TEST_TAG }} --version - - name: Create tags for publishing image id: meta uses: docker/metadata-action@v5