From 15b0cc89baa15bdbd8bb14a4578b7b7ba01c35c6 Mon Sep 17 00:00:00 2001 From: Dragomir Penev <6687393+dragomirp@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:57:05 +0200 Subject: [PATCH 1/6] patch(integration_test_charm.yaml): Remove dotnet cleanup (#261) Remove dotnet cleanup --- .github/workflows/integration_test_charm.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration_test_charm.yaml b/.github/workflows/integration_test_charm.yaml index 6f9b5134..9fd80a78 100644 --- a/.github/workflows/integration_test_charm.yaml +++ b/.github/workflows/integration_test_charm.yaml @@ -218,7 +218,6 @@ jobs: printf '\nDisk usage before cleanup\n' df --human-readable # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - rm -r /usr/share/dotnet rm -r /opt/hostedtoolcache/ printf '\nDisk usage after cleanup\n' df --human-readable From 349e0c80728a82041637a75cfb16c135fb2f071c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 6 Jan 2025 15:00:29 +0000 Subject: [PATCH 2/6] patch: Replace `sg -c` with `sudo --user` (#259) `sg lxd -c` on Ubuntu 24.04 prompts for password: https://github.com/actions/runner-images/issues/9932 Best guess is that this change was caused by [shadow](https://github.com/shadow-maint/shadow) being upgraded from 4.8.1 to 4.13 (from Ubuntu 22.04 to 24.04) The contents of `/etc/gshadow` do not appear to have changed from Ubuntu 22.04 to 24.04. i.e. ``` # Ubuntu 22.04 GitHub runner $ sudo cat /etc/gshadow | grep lxd lxd:!::runneradmin ``` ``` # Ubuntu 24.04 GitHub runner $ sudo cat /etc/gshadow | grep lxd lxd:!::runneradmin ``` --- .github/workflows/build_charm.yaml | 10 ++-- .github/workflows/build_rock.yaml | 8 +-- .github/workflows/build_snap.yaml | 8 +-- .github/workflows/integration_test_charm.yaml | 59 ++++++++----------- 4 files changed, 39 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 6cac8df8..7a6b2fb6 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -122,9 +122,9 @@ jobs: sudo snap refresh lxd ${{ steps.lxd-snap-version.outputs.install_flag }} fi sudo adduser "$USER" lxd - # `newgrp` does not work in GitHub Actions; use `sg` instead - sg lxd -c "lxd waitready" - sg lxd -c "lxd init --auto" + # `newgrp` does not work in GitHub Actions; use `sudo --user` instead + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd waitready + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd init --auto # Workaround for Docker & LXD on same machine sudo iptables -F FORWARD sudo iptables -P FORWARD ACCEPT @@ -143,12 +143,12 @@ jobs: run: | if '${{ inputs.cache }}' then - sg lxd -c "charmcraftcache pack -v --bases-index='${{ matrix.base.id }}'" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- charmcraftcache pack -v --bases-index='${{ matrix.base.id }}' else # Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2 touch requirements.txt - sg lxd -c "charmcraft pack -v --bases-index='${{ matrix.base.id }}'" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- charmcraft pack -v --bases-index='${{ matrix.base.id }}' fi env: # Used by charmcraftcache (to avoid GitHub API rate limit) diff --git a/.github/workflows/build_rock.yaml b/.github/workflows/build_rock.yaml index a46fa4a3..bd8c9720 100644 --- a/.github/workflows/build_rock.yaml +++ b/.github/workflows/build_rock.yaml @@ -81,9 +81,9 @@ jobs: - name: Set up environment run: | sudo adduser "$USER" lxd - # `newgrp` does not work in GitHub Actions; use `sg` instead - sg lxd -c "lxd waitready" - sg lxd -c "lxd init --auto" + # `newgrp` does not work in GitHub Actions; use `sudo --user` instead + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd waitready + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd init --auto # Workaround for Docker & LXD on same machine sudo iptables -F FORWARD sudo iptables -P FORWARD ACCEPT @@ -93,7 +93,7 @@ jobs: - name: Pack rock id: pack working-directory: ${{ inputs.path-to-rock-directory }} - run: sg lxd -c "rockcraft pack -v --platform='${{ matrix.base.id }}'" + run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- rockcraft pack -v --platform='${{ matrix.base.id }}' - name: Upload rockcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/build_snap.yaml b/.github/workflows/build_snap.yaml index d7e46faa..437c5a75 100644 --- a/.github/workflows/build_snap.yaml +++ b/.github/workflows/build_snap.yaml @@ -84,9 +84,9 @@ jobs: - name: Set up environment run: | sudo adduser "$USER" lxd - # `newgrp` does not work in GitHub Actions; use `sg` instead - sg lxd -c "lxd waitready" - sg lxd -c "lxd init --auto" + # `newgrp` does not work in GitHub Actions; use `sudo --user` instead + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd waitready + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd init --auto # Workaround for Docker & LXD on same machine sudo iptables -F FORWARD sudo iptables -P FORWARD ACCEPT @@ -96,7 +96,7 @@ jobs: - name: Pack snap id: pack working-directory: ${{ inputs.path-to-snap-project-directory }} - run: sg lxd -c "snapcraft pack -v --build-for='${{ matrix.base.id }}'" + run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- snapcraft pack -v --build-for='${{ matrix.base.id }}' - name: Upload snapcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/integration_test_charm.yaml b/.github/workflows/integration_test_charm.yaml index 9fd80a78..4febe80d 100644 --- a/.github/workflows/integration_test_charm.yaml +++ b/.github/workflows/integration_test_charm.yaml @@ -261,31 +261,22 @@ jobs: run: add-ssh-keys env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Parse cloud input + - name: Validate cloud input timeout-minutes: 1 - id: parse-cloud shell: python # Keep synchronized with inputs.cloud description run: | - import json - import os - CLOUD = "${{ inputs.cloud }}" if CLOUD == "lxd": - group = "lxd" + pass elif CLOUD == "microk8s": SNAP_CHANNEL = "${{ inputs.microk8s-snap-channel }}" assert ( SNAP_CHANNEL != "" ), '`microk8s-snap-channel` input required if `cloud` is "microk8s"' assert "strict" in SNAP_CHANNEL.lower(), "Only strict microk8s snap supported" - group = "snap_microk8s" else: raise ValueError(f"`cloud` input not recognized: {CLOUD}") - output = f"group={group}" - print(output) - with open(os.environ["GITHUB_OUTPUT"], "a") as file: - file.write(output) - name: Parse Juju agent version & snap channel timeout-minutes: 1 id: parse-versions @@ -327,11 +318,11 @@ jobs: if: ${{ inputs.cloud == 'lxd' }} run: | sudo snap refresh lxd --channel='${{ inputs.lxd-snap-channel }}' - sudo adduser "$USER" '${{ steps.parse-cloud.outputs.group }}' - # `newgrp` does not work in GitHub Actions; use `sg` instead - sg '${{ steps.parse-cloud.outputs.group }}' -c "lxd waitready" - sg '${{ steps.parse-cloud.outputs.group }}' -c "lxd init --auto" - sg '${{ steps.parse-cloud.outputs.group }}' -c "lxc network set lxdbr0 ipv6.address none" + sudo adduser "$USER" lxd + # `newgrp` does not work in GitHub Actions; use `sudo --user` instead + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd waitready + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxd init --auto + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxc network set lxdbr0 ipv6.address none sudo iptables -F FORWARD sudo iptables -P FORWARD ACCEPT - name: Set up microk8s @@ -341,14 +332,14 @@ jobs: sudo apt-get update sudo apt-get install retry -y sudo snap install microk8s --channel='${{ inputs.microk8s-snap-channel }}' - sudo adduser "$USER" '${{ steps.parse-cloud.outputs.group }}' + sudo adduser "$USER" snap_microk8s - name: (IS hosted) Configure microk8s Docker Hub mirror timeout-minutes: 5 if: ${{ inputs.cloud == 'microk8s' && matrix.groups.is_hosted }} run: | # Wait for microk8s to populate iptables # https://chat.canonical.com/canonical/pl/jo5cg6wqjjrudqd5ybj6hhttee - sg '${{ steps.parse-cloud.outputs.group }}' -c "microk8s status --wait-ready" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready sudo tee /var/snap/microk8s/current/args/certs.d/docker.io/hosts.toml << EOF server = "$DOCKERHUB_MIRROR" @@ -362,21 +353,23 @@ jobs: timeout-minutes: 15 if: ${{ inputs.cloud == 'microk8s' }} run: | - # `newgrp` does not work in GitHub Actions; use `sg` instead - sg '${{ steps.parse-cloud.outputs.group }}' -c "microk8s status --wait-ready" - sg '${{ steps.parse-cloud.outputs.group }}' -c "retry --times 3 --delay 5 -- sudo microk8s enable dns" - sg '${{ steps.parse-cloud.outputs.group }}' -c "microk8s status --wait-ready" - sg '${{ steps.parse-cloud.outputs.group }}' -c "microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/coredns" - sg '${{ steps.parse-cloud.outputs.group }}' -c "retry --times 3 --delay 5 -- sudo microk8s enable hostpath-storage" - sg '${{ steps.parse-cloud.outputs.group }}' -c "microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/hostpath-provisioner" + # `newgrp` does not work in GitHub Actions; use `sudo --user` instead + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable dns + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s status --wait-ready + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/coredns + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable hostpath-storage + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/hostpath-provisioner if [[ '${{ inputs.metallb-addon }}' = 'true' ]] then IPADDR=$(ip -4 -j route get 2.2.2.2 | sed -n -e 's/^.*prefsrc\":"\([^ "]*\).*/\1/p') - sg '${{ steps.parse-cloud.outputs.group }}' -c "retry --times 3 --delay 5 -- sudo microk8s enable metallb:$IPADDR-$IPADDR" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- retry --times 3 --delay 5 -- sudo microk8s enable "metallb:$IPADDR-$IPADDR" fi mkdir ~/.kube/ # Used by lightkube and kubernetes (Python package) - sg '${{ steps.parse-cloud.outputs.group }}' -c "microk8s config > ~/.kube/config" + # shellcheck disable=SC2024 + # (`microk8s config` needs to be run with sudo, but writing to `~/.kube/config` does not) + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- microk8s config > ~/.kube/config # Temporarily collect logs to debug intermittent issue # https://github.com/canonical/data-platform-workflows/issues/217 - name: Collect microk8s logs for debugging @@ -408,18 +401,18 @@ jobs: # (shellcheck sees it as constant, but GitHub Actions expression is not constant between workflow runs) if [[ '${{ steps.parse-versions.outputs.snap_channel }}' == 2.* ]] then - sg '${{ steps.parse-cloud.outputs.group }}' -c "lxc image copy ubuntu:8de71f421b30 local: --alias 'juju/focal/amd64'" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxc image copy ubuntu:8de71f421b30 local: --alias 'juju/focal/amd64' else - sg '${{ steps.parse-cloud.outputs.group }}' -c "lxc image copy ubuntu:82b997ec581b local: --alias 'juju/ubuntu@22.04/amd64'" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxc image copy ubuntu:82b997ec581b local: --alias 'juju/ubuntu@22.04/amd64' fi else - sg '${{ steps.parse-cloud.outputs.group }}' -c "lxc image copy ubuntu:60d56aa663ed local: --alias 'juju/ubuntu@22.04/arm64'" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxc image copy ubuntu:60d56aa663ed local: --alias 'juju/ubuntu@22.04/arm64' fi - name: Set up environment timeout-minutes: 15 run: | mkdir -p ~/.local/share/juju # Workaround for juju 3 strict snap - sg '${{ steps.parse-cloud.outputs.group }}' -c "juju bootstrap '${{ inputs.cloud }}' --config model-logs-size=10G '${{ steps.parse-versions.outputs.agent_bootstrap_option }}'" + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- juju bootstrap '${{ inputs.cloud }}' --config model-logs-size=10G '${{ steps.parse-versions.outputs.agent_bootstrap_option }}' juju model-defaults logging-config='=INFO; unit=DEBUG' juju add-model test # Unable to set constraint on all models because of Juju bug: @@ -431,7 +424,7 @@ jobs: - name: lxc image list timeout-minutes: 1 if: ${{ inputs.cloud == 'lxd' }} - run: sg '${{ steps.parse-cloud.outputs.group }}' -c "lxc image list" + run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- lxc image list - name: Update python-libjuju version timeout-minutes: 3 if: ${{ inputs.libjuju-version-constraint }} @@ -474,7 +467,7 @@ jobs: - name: Run integration tests timeout-minutes: 120 id: tests - run: sg '${{ steps.parse-cloud.outputs.group }}' -c "tox run -e integration -- '${{ matrix.groups.path_to_test_file }}' --group='${{ matrix.groups.group_id }}' -m '${{ steps.select-test-stability.outputs.mark_expression }}' --model test ${{ steps.allure-option.outputs.option }}" + run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- tox run -e integration -- '${{ matrix.groups.path_to_test_file }}' --group='${{ matrix.groups.group_id }}' -m '${{ steps.select-test-stability.outputs.mark_expression }}' --model test ${{ steps.allure-option.outputs.option }} env: SECRETS_FROM_GITHUB: ${{ secrets.integration-test }} - name: (beta) Upload Allure results From 078fb649213bbffe223e826f04560c2e9c9c3396 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 6 Jan 2025 15:02:15 +0000 Subject: [PATCH 3/6] patch(build_charm.yaml): Support poetry v2 (#260) warnings.export setting removed in poetry v2 --- .github/workflows/build_charm.yaml | 2 -- .github/workflows/integration_test_charm.md | 12 ------------ 2 files changed, 14 deletions(-) diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 7a6b2fb6..5f6975c4 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -132,8 +132,6 @@ jobs: sudo snap install charmcraft --classic ${{ steps.charmcraft-snap-version.outputs.install_flag }} pipx install poetry pipx inject poetry poetry-plugin-export - # TODO: Remove after https://github.com/python-poetry/poetry/pull/5980 is closed - poetry config warnings.export false pipx install charmcraftcache - run: snap list diff --git a/.github/workflows/integration_test_charm.md b/.github/workflows/integration_test_charm.md index 45c5ad33..ca36a8cd 100644 --- a/.github/workflows/integration_test_charm.md +++ b/.github/workflows/integration_test_charm.md @@ -33,18 +33,6 @@ pytest-operator-groups = {git = "https://github.com/canonical/data-platform-work to your integration test dependencies in `pyproject.toml`. #### Step B -Disable Poetry's parallel installation for integration test dependencies. - -Example `tox.ini`: -```ini -[testenv:integration] -set_env = - {[testenv]set_env} - # Workaround for https://github.com/python-poetry/poetry/issues/6958 - POETRY_INSTALLER_PARALLEL = false -``` - -#### Step C If you're using tox, pass in the `CI` and `GITHUB_OUTPUT` environment variables in `tox.ini`. ```ini [testenv:integration] From 11c673f692893a15d15ee63469420e91f91f8a95 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 7 Jan 2025 11:50:20 +0000 Subject: [PATCH 4/6] patch: Install lxd snap on runner (#262) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ubuntu-24.04 image, unlike 22.04 image, does not have lxd snap pre-installed e.g. Ubuntu 22.04 GitHub runner ``` $ snap list Name Version Rev Tracking Publisher Notes core20 20240705 2379 latest/stable canonical✓ base lxd 5.0.3-80aeff7 29351 5.0/stable/… canonical✓ - snapd 2.63 21759 latest/stable canonical✓ snapd ``` Ubuntu 24.04 GitHub runner ``` $ snap list No snaps are installed yet. Try 'snap install hello-world'. ``` --- .github/workflows/build_charm.yaml | 4 +--- .github/workflows/integration_test_charm.yaml | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 5f6975c4..85b4cd18 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -112,9 +112,7 @@ jobs: uses: actions/checkout@v4 - name: Set up environment run: | - # Placeholder (so that shellcheck disable does not apply to entire script) - # https://github.com/koalaman/shellcheck/issues/960#issuecomment-318918175 - true + sudo snap install lxd ${{ steps.lxd-snap-version.outputs.install_flag }} # shellcheck disable=SC2078 # (shellcheck sees it as constant, but GitHub Actions expression is not constant between workflow runs) if [[ '${{ steps.lxd-snap-version.outputs.install_flag }}' ]] diff --git a/.github/workflows/integration_test_charm.yaml b/.github/workflows/integration_test_charm.yaml index 4febe80d..ccce455e 100644 --- a/.github/workflows/integration_test_charm.yaml +++ b/.github/workflows/integration_test_charm.yaml @@ -317,6 +317,7 @@ jobs: timeout-minutes: 5 if: ${{ inputs.cloud == 'lxd' }} run: | + sudo snap install lxd --channel='${{ inputs.lxd-snap-channel }}' sudo snap refresh lxd --channel='${{ inputs.lxd-snap-channel }}' sudo adduser "$USER" lxd # `newgrp` does not work in GitHub Actions; use `sudo --user` instead From eb08b369840416f5ef5b5277367ffcb3906b84d4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 10 Jan 2025 10:00:18 +0000 Subject: [PATCH 5/6] breaking: Revert "Change default charmcraft version to 2.x (#219)" (#257) This reverts commit da8bb57d6a9650fef8683cb76f233cd672f63db5. --- python/cli/data_platform_workflows_cli/parse_snap_version.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/cli/data_platform_workflows_cli/parse_snap_version.py b/python/cli/data_platform_workflows_cli/parse_snap_version.py index c088b918..29c06615 100644 --- a/python/cli/data_platform_workflows_cli/parse_snap_version.py +++ b/python/cli/data_platform_workflows_cli/parse_snap_version.py @@ -17,8 +17,6 @@ def main(): install_flag = f"'--revision={args.revision}'" elif args.channel: install_flag = f"'--channel={args.channel}'" - elif args.channel_input_name == "charmcraft-snap-channel": - install_flag = "'--channel=2.x/stable'" else: install_flag = None github_actions.output["install_flag"] = install_flag From 92d0a9f28a22c57b5965866c22f65b5021d3b115 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 10 Jan 2025 10:07:27 +0000 Subject: [PATCH 6/6] breaking: Switch to ST124 "shorthand notation" syntax in charmcraft.yaml (#258) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enables building & releasing multi-base charms with 24.04 in a single charmcraft.yaml and git branch Integration testing is not supported on multiple bases—it is currently only supported on 22.04 ## Breaking changes - ST124 "shorthand notation" syntax required in charmcraft.yaml - `pytest-operator-cache` (which overrides `ops_test.build_charm` from pytest-operator) now assumes 22.04 base ## Deprecation notice - `pytest-operator-cache` is deprecated & will be removed in a future release --- .github/workflows/build_charm.md | 16 +++ .github/workflows/build_charm.yaml | 33 +++--- .github/workflows/build_rock.md | 14 ++- .github/workflows/build_rock.yaml | 22 ++-- .github/workflows/build_snap.md | 33 +++++- .github/workflows/build_snap.yaml | 26 ++-- .github/workflows/integration_test_charm.md | 4 +- .../craft_tools/charmcraft_platforms.py | 54 +++++++++ .../craft_tools/collect_bases.py | 112 ------------------ .../craft_tools/collect_platforms.py | 100 ++++++++++++++++ .../craft_tools/release.py | 5 - python/cli/pyproject.toml | 6 +- .../pytest_operator_cache/README.md | 3 + .../deprecation_notice.md | 14 +++ .../pytest_operator_cache/_plugin.py | 43 +++---- 15 files changed, 292 insertions(+), 193 deletions(-) create mode 100644 python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py delete mode 100644 python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py create mode 100644 python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py create mode 100644 python/pytest_plugins/pytest_operator_cache/deprecation_notice.md diff --git a/.github/workflows/build_charm.md b/.github/workflows/build_charm.md index 33e0bcae..a8efcd96 100644 --- a/.github/workflows/build_charm.md +++ b/.github/workflows/build_charm.md @@ -16,3 +16,19 @@ with: cache: true ``` remember to add your charm's branch(es) to charmcraftcache by running `ccc add` or by [opening an issue](https://github.com/canonical/charmcraftcache-hub/issues/new?assignees=&labels=add-charm&projects=&template=add_charm_branch.yaml&title=Add+charm+branch). + +### Required charmcraft.yaml syntax +Only [ST124 - Multi-base platforms in craft tools](https://docs.google.com/document/d/1QVHxZumruKVZ3yJ2C74qWhvs-ye5I9S6avMBDHs2YcQ/edit) "shorthand notation" syntax is supported + +#### Example +```yaml +platforms: + ubuntu@22.04:amd64: + ubuntu@22.04:arm64: + ubuntu@24.04:amd64: + ubuntu@24.04:arm64: +``` + +Under the charmcraft.yaml `platforms` key, `build-on` and `build-for` syntax are not supported + +The `base` and `bases` charmcraft.yaml keys are not supported diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 85b4cd18..6d016aac 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -51,11 +51,11 @@ on: outputs: artifact-prefix: description: Charm packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: - name: Collect bases for charm | ${{ inputs.path-to-charm-directory }} + collect-platforms: + name: Collect platforms for charm | ${{ inputs.path-to-charm-directory }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -70,25 +70,25 @@ jobs: run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - name: Checkout uses: actions/checkout@v4 - - name: Collect charm bases to build from charmcraft.yaml + - name: Collect charm platforms to build from charmcraft.yaml id: collect - run: collect-charm-bases --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}' + run: collect-charm-platforms --directory='${{ inputs.path-to-charm-directory }}' --cache='${{ inputs.cache }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build charm | base #${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build charm | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 120 steps: - name: (GitHub-hosted ARM runner) Install libpq-dev - if: ${{ matrix.base.runner == 'Ubuntu_ARM64_4C_16G_02' }} + if: ${{ matrix.platform.runner == 'Ubuntu_ARM64_4C_16G_02' }} # Needed for `charmcraftcache` to resolve dependencies (for postgresql charms with psycopg2) run: | sudo apt-get update @@ -139,12 +139,9 @@ jobs: run: | if '${{ inputs.cache }}' then - sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- charmcraftcache pack -v --bases-index='${{ matrix.base.id }}' + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- charmcraftcache pack -v --platform='${{ matrix.platform.name }}' else - # Workaround for https://github.com/canonical/charmcraft/issues/1389 on charmcraft 2 - touch requirements.txt - - sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- charmcraft pack -v --bases-index='${{ matrix.base.id }}' + sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- charmcraft pack -v --platform='${{ matrix.platform.name }}' fi env: # Used by charmcraftcache (to avoid GitHub API rate limit) @@ -153,14 +150,14 @@ jobs: if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-charmcraft-build-${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }} + name: logs-charmcraft-build-${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }} path: ~/.local/state/charmcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload charm package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-base-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name_in_artifact }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/.github/workflows/build_rock.md b/.github/workflows/build_rock.md index c933765b..89d2eaa4 100644 --- a/.github/workflows/build_rock.md +++ b/.github/workflows/build_rock.md @@ -8,4 +8,16 @@ jobs: build: name: Build rock uses: canonical/data-platform-workflows/.github/workflows/build_rock.yaml@v0.0.0 -``` \ No newline at end of file +``` + +### Supported `platforms` syntax in rockcraft.yaml +Only "shorthand notation" is supported + +Example rockcraft.yaml +```yaml +platforms: + amd64: + arm64: +``` + +`build-on` and `build-for` are not supported diff --git a/.github/workflows/build_rock.yaml b/.github/workflows/build_rock.yaml index bd8c9720..d00392c5 100644 --- a/.github/workflows/build_rock.yaml +++ b/.github/workflows/build_rock.yaml @@ -28,10 +28,10 @@ on: outputs: artifact-prefix: description: Rock packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: + collect-platforms: name: Collect platforms for rock | ${{ inputs.path-to-rock-directory }} runs-on: ubuntu-latest timeout-minutes: 5 @@ -49,19 +49,19 @@ jobs: uses: actions/checkout@v4 - name: Collect rock platforms to build from rockcraft.yaml id: collect - run: collect-rock-bases --directory='${{ inputs.path-to-rock-directory }}' + run: collect-rock-platforms --directory='${{ inputs.path-to-rock-directory }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build rock | ${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build rock | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 15 steps: - name: Get workflow version @@ -93,19 +93,19 @@ jobs: - name: Pack rock id: pack working-directory: ${{ inputs.path-to-rock-directory }} - run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- rockcraft pack -v --platform='${{ matrix.base.id }}' + run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- rockcraft pack -v --platform='${{ matrix.platform.name }}' - name: Upload rockcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-rockcraft-build-${{ inputs.artifact-prefix }}-architecture-${{ matrix.base.id }} + name: logs-rockcraft-build-${{ inputs.artifact-prefix }}-platform-${{ matrix.platform.name }} path: ~/.local/state/rockcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload rock package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-architecture-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/.github/workflows/build_snap.md b/.github/workflows/build_snap.md index 1ac88e00..181cf6a8 100644 --- a/.github/workflows/build_snap.md +++ b/.github/workflows/build_snap.md @@ -8,4 +8,35 @@ jobs: build: name: Build snap uses: canonical/data-platform-workflows/.github/workflows/build_snap.yaml@v0.0.0 -``` \ No newline at end of file +``` + +### Supported `platforms` and `architectures` syntax in snapcraft.yaml +See https://snapcraft.io/docs/architectures#how-to-create-a-snap-for-a-specific-architecture + +#### core24 +Only `platforms` is supported. `architectures` is not supported + +Only "shorthand notation" is supported + +Example snapcraft.yaml +```yaml +platforms: + amd64: + arm64: +``` + +`build-on` and `build-for` are not supported + +#### core22 +Only `architectures` is supported. `platforms` is not supported + +`architectures` must be a list of dictionaries. Each dictionary in the list must contain a `build-on` key + +Example snapcraft.yaml +```yaml +architectures: + - build-on: [amd64] + build-for: [amd64] + - build-on: [arm64] + build-for: [arm64] +``` diff --git a/.github/workflows/build_snap.yaml b/.github/workflows/build_snap.yaml index 437c5a75..740484f1 100644 --- a/.github/workflows/build_snap.yaml +++ b/.github/workflows/build_snap.yaml @@ -31,11 +31,11 @@ on: outputs: artifact-prefix: description: Snap packages are uploaded to GitHub artifacts beginning with this prefix - value: ${{ jobs.collect-bases.outputs.artifact-prefix-with-inputs }} + value: ${{ jobs.collect-platforms.outputs.artifact-prefix-with-inputs }} jobs: - collect-bases: - name: Collect architectures for snap | ${{ inputs.path-to-snap-project-directory }} + collect-platforms: + name: Collect platforms for snap | ${{ inputs.path-to-snap-project-directory }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -50,21 +50,21 @@ jobs: run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli - name: Checkout uses: actions/checkout@v4 - - name: Collect snap architectures to build from snapcraft.yaml + - name: Collect snap platforms to build from snapcraft.yaml id: collect - run: collect-snap-bases --directory='${{ inputs.path-to-snap-project-directory }}' + run: collect-snap-platforms --directory='${{ inputs.path-to-snap-project-directory }}' outputs: - bases: ${{ steps.collect.outputs.bases }} + platforms: ${{ steps.collect.outputs.platforms }} artifact-prefix-with-inputs: ${{ inputs.artifact-prefix || steps.collect.outputs.default_prefix }} build: strategy: matrix: - base: ${{ fromJSON(needs.collect-bases.outputs.bases) }} - name: 'Build snap | ${{ matrix.base.id }}' + platform: ${{ fromJSON(needs.collect-platforms.outputs.platforms) }} + name: 'Build snap | ${{ matrix.platform.name }}' needs: - - collect-bases - runs-on: ${{ matrix.base.runner }} + - collect-platforms + runs-on: ${{ matrix.platform.runner }} timeout-minutes: 30 steps: - name: Get workflow version @@ -96,19 +96,19 @@ jobs: - name: Pack snap id: pack working-directory: ${{ inputs.path-to-snap-project-directory }} - run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- snapcraft pack -v --build-for='${{ matrix.base.id }}' + run: sudo --user "$USER" --preserve-env --preserve-env=PATH -- env -- snapcraft pack -v --build-for='${{ matrix.platform.name }}' - name: Upload snapcraft logs if: ${{ failure() && steps.pack.outcome == 'failure' }} uses: actions/upload-artifact@v4 with: - name: logs-snapcraft-build-${{ inputs.artifact-prefix }}-architecture-${{ matrix.base.id }} + name: logs-snapcraft-build-${{ inputs.artifact-prefix }}-platform-${{ matrix.platform.name }} path: ~/.local/state/snapcraft/log/ if-no-files-found: error - run: touch .empty - name: Upload snap package uses: actions/upload-artifact@v4 with: - name: ${{ needs.collect-bases.outputs.artifact-prefix-with-inputs }}-architecture-${{ matrix.base.id }} + name: ${{ needs.collect-platforms.outputs.artifact-prefix-with-inputs }}-platform-${{ matrix.platform.name }} # .empty file required to preserve directory structure # See https://github.com/actions/upload-artifact/issues/344#issuecomment-1379232156 path: | diff --git a/.github/workflows/integration_test_charm.md b/.github/workflows/integration_test_charm.md index ca36a8cd..b75c1825 100644 --- a/.github/workflows/integration_test_charm.md +++ b/.github/workflows/integration_test_charm.md @@ -1,5 +1,8 @@ Workflow file: [integration_test_charm.yaml](integration_test_charm.yaml) +> [!WARNING] +> The `pytest-operator-cache` plugin is **deprecated**. Follow the migration instructions here: [pytest_operator_cache/deprecation_notice.md](../../python/pytest_plugins/pytest_operator_cache/deprecation_notice.md) + ## Usage ### Step 1: Create your workflow ```yaml @@ -27,7 +30,6 @@ jobs: #### Step A Add ```toml -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v0.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"} pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v0.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"} ``` to your integration test dependencies in `pyproject.toml`. diff --git a/python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py new file mode 100644 index 00000000..4650bbc1 --- /dev/null +++ b/python/cli/data_platform_workflows_cli/craft_tools/charmcraft_platforms.py @@ -0,0 +1,54 @@ +# Copied from https://github.com/canonical/charmcraftcache/blob/main/charmcraftcache/_platforms.py +import pathlib + +import yaml + +_SYNTAX_DOCS = "https://github.com/canonical/data-platform-workflows/blob/main/.github/workflows/build_charm.md#required-charmcraftyaml-syntax" + + +class Platform(str): + """Platform in charmcraft.yaml 'platforms' (e.g. 'ubuntu@22.04:amd64')""" + + def __new__(cls, value: str, /): + try: + _, architecture = value.split(":") + except ValueError: + raise ValueError( + "Invalid ST124 shorthand notation in charmcraft.yaml 'platforms': " + f"{repr(value)}\n\nMore info: {_SYNTAX_DOCS}" + ) + instance: Platform = super().__new__(cls, value) + instance.architecture = architecture + return instance + + +def get(charmcraft_yaml: pathlib.Path, /): + """Get platforms from charmcraft.yaml""" + charmcraft_yaml_data = yaml.safe_load(charmcraft_yaml.read_text()) + for key in ("base", "bases"): + if key in charmcraft_yaml_data: + raise ValueError( + f"'{key}' key in charmcraft.yaml not supported. Use 'platforms' key instead.\n\n" + f"More info: {_SYNTAX_DOCS}" + ) + yaml_platforms = charmcraft_yaml_data.get("platforms") + if not yaml_platforms: + raise ValueError( + f"'platforms' key in charmcraft.yaml required\n\nMore info: {_SYNTAX_DOCS}" + ) + if not isinstance(yaml_platforms, dict): + raise TypeError( + "Expected charmcraft.yaml 'platforms' with type 'dict', got " + f"{repr(type(yaml_platforms).__name__)}: {repr(yaml_platforms)}\n\n" + f"More info: {_SYNTAX_DOCS}" + ) + for value in yaml_platforms.values(): + if value is not None: + raise ValueError( + "Shorthand notation required ('build-on' and 'build-for' not supported) in " + f"charmcraft.yaml 'platforms'.\n\nMore info: {_SYNTAX_DOCS}" + ) + # Validate format of keys in `yaml_platforms` + platforms = [Platform(platform) for platform in yaml_platforms] + + return platforms diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py deleted file mode 100644 index 22c6d85a..00000000 --- a/python/cli/data_platform_workflows_cli/craft_tools/collect_bases.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. -"""Collect bases to build - -charmcraft: "bases" -snapcraft: "architectures" -rockcraft: "platforms" - -snaps & rocks are usually built on multiple architectures but only one Ubuntu version/base -charms (subordinate) can be built on multiple Ubuntu versions -""" - -import argparse -import json -import logging -import pathlib -import sys - -import yaml - -from .. import github_actions -from . import craft - -logging.basicConfig(level=logging.INFO, stream=sys.stdout) -RUNNERS = { - craft.Architecture.X64: "ubuntu-latest", - craft.Architecture.ARM64: "Ubuntu_ARM64_4C_16G_02", -} - - -def get_bases(*, craft_: craft.Craft, yaml_data): - """Get architecture for each base - - For charms, multiple bases can have the same architecture - (e.g. Ubuntu 20.04 X64 and Ubuntu 22.04 X64) - - For snaps & rocks, the Ubuntu version is the same for all architectures. - """ - if craft_ is craft.Craft.ROCK: - # https://canonical-rockcraft.readthedocs-hosted.com/en/latest/reference/rockcraft.yaml/#platforms - return [craft.Architecture(arch) for arch in yaml_data["platforms"]] - if craft_ is craft.Craft.SNAP: - bases = yaml_data.get("architectures") - if not bases: - # Default to X64 - return [craft.Architecture.X64] - elif craft_ is craft.Craft.CHARM: - bases = yaml_data["bases"] - else: - raise ValueError - arch_for_bases = [] - for platform in bases: - if craft_ is craft.Craft.SNAP: - # https://snapcraft.io/docs/explanation-architectures - build_on_architectures = platform["build-on"] - elif craft_ is craft.Craft.CHARM: - # https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713 - build_on = platform.get("build-on") - if build_on: - assert isinstance(build_on, list) and len(build_on) == 1 - platform = build_on[0] - build_on_architectures = platform.get("architectures") - if not build_on_architectures: - # Default to X64 - arch_for_bases.append(craft.Architecture.X64) - continue - else: - raise ValueError - assert ( - len(build_on_architectures) == 1 - ), f"Multiple architectures ({build_on_architectures}) in one ({craft_.value}craft.yaml) base/architecture entry not supported. Use one entry per architecture" - arch_for_bases.append(craft.Architecture(build_on_architectures[0])) - return arch_for_bases - - -def collect(craft_: craft.Craft): - """Collect bases to build from *craft.yaml""" - parser = argparse.ArgumentParser() - parser.add_argument("--directory", required=True) - if craft_ is craft.Craft.CHARM: - parser.add_argument("--cache", required=True) - args = parser.parse_args() - craft_file = pathlib.Path(args.directory, f"{craft_.value}craft.yaml") - if craft_ is craft.Craft.SNAP: - craft_file = craft_file.parent / "snap" / craft_file.name - yaml_data = yaml.safe_load(craft_file.read_text()) - bases_ = get_bases(craft_=craft_, yaml_data=yaml_data) - bases = [] - for index, architecture in enumerate(bases_): - # id used to select base in `*craft pack` - if craft_ is craft.Craft.CHARM: - id_ = index - else: - id_ = architecture.value - bases.append({"id": id_, "runner": RUNNERS[architecture]}) - github_actions.output["bases"] = json.dumps(bases) - default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' - if craft_ is craft.Craft.CHARM: - default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' - github_actions.output["default_prefix"] = default_prefix - - -def snap(): - collect(craft.Craft.SNAP) - - -def rock(): - collect(craft.Craft.ROCK) - - -def charm(): - collect(craft.Craft.CHARM) diff --git a/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py new file mode 100644 index 00000000..385cb72e --- /dev/null +++ b/python/cli/data_platform_workflows_cli/craft_tools/collect_platforms.py @@ -0,0 +1,100 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +"""Collect platforms to build + +charmcraft: Only ST124 shorthand notation `platforms` are supported +snapcraft: (ST124 not supported) core22 `architectures` and core24 shorthand `platforms` supported +rockcraft: (ST124 not supported) shorthand `platforms` supported + +snaps & rocks are usually built on multiple architectures but only one Ubuntu version/base +charms (subordinate) can be built on multiple Ubuntu versions +""" + +import argparse +import json +import logging +import pathlib +import sys + +import yaml + +from .. import github_actions +from . import craft +from . import charmcraft_platforms + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) +RUNNERS = { + craft.Architecture.X64: "ubuntu-latest", + craft.Architecture.ARM64: "Ubuntu_ARM64_4C_16G_02", +} + + +def collect(craft_: craft.Craft): + """Collect platforms to build from *craft.yaml""" + parser = argparse.ArgumentParser() + parser.add_argument("--directory", required=True) + if craft_ is craft.Craft.CHARM: + parser.add_argument("--cache", required=True) + args = parser.parse_args() + craft_file = pathlib.Path(args.directory, f"{craft_.value}craft.yaml") + if craft_ is craft.Craft.SNAP: + craft_file = craft_file.parent / "snap" / craft_file.name + yaml_data = yaml.safe_load(craft_file.read_text()) + platforms = [] + if craft_ is craft.Craft.CHARM: + for platform in charmcraft_platforms.get(craft_file): + # Example `platform`: "ubuntu@22.04:amd64" + platforms.append( + { + "name": platform, + "runner": RUNNERS[platform.architecture], + "name_in_artifact": platform.replace(":", "-"), + } + ) + elif craft_ is craft.Craft.ROCK: + for platform in yaml_data["platforms"]: + # Example `platform`: "amd64" + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + elif craft_ is craft.Craft.SNAP: + if yaml_data["base"] == "core24": + platforms_ = yaml_data["platforms"] + if not isinstance(platforms_, dict): + raise TypeError("Expected type 'dict' for snapcraft.yaml 'platforms'") + for value in platforms_.values(): + if value is not None: + raise ValueError( + "Only shorthand notation supported in snapcraft.yaml 'platforms'. " + "'build-on' and 'build-for' not supported" + ) + for platform in platforms_: + # Example `platform`: "amd64" + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + elif yaml_data["base"] == "core22": + for entry in yaml_data["architectures"]: + # Example: "amd64" + platform = entry["build-on"] + architecture = craft.Architecture(platform) + platforms.append({"name": platform, "runner": RUNNERS[architecture]}) + else: + raise ValueError(f'Unsupported snapcraft.yaml base: {repr(yaml_data["base"])}') + else: + raise ValueError + github_actions.output["platforms"] = json.dumps(platforms) + default_prefix = f'packed-{craft_.value}-{args.directory.replace("/", "-")}' + if craft_ is craft.Craft.CHARM: + default_prefix = f'packed-{craft_.value}-cache-{args.cache}-{args.directory.replace("/", "-")}' + github_actions.output["default_prefix"] = default_prefix + + +def snap(): + collect(craft.Craft.SNAP) + + +def rock(): + collect(craft.Craft.ROCK) + + +def charm(): + collect(craft.Craft.CHARM) diff --git a/python/cli/data_platform_workflows_cli/craft_tools/release.py b/python/cli/data_platform_workflows_cli/craft_tools/release.py index 80f6edb5..2a780945 100644 --- a/python/cli/data_platform_workflows_cli/craft_tools/release.py +++ b/python/cli/data_platform_workflows_cli/craft_tools/release.py @@ -193,11 +193,6 @@ def rock(): def charm(): - # Remove `charmcraft.yaml` from working directory (directory that subprocess will run as) if it - # exists. - # Workaround for https://github.com/canonical/charmcraft/issues/1389 - pathlib.Path("charmcraft.yaml").unlink(missing_ok=True) - parser = argparse.ArgumentParser() parser.add_argument("--directory", required=True) parser.add_argument("--channel", required=True) diff --git a/python/cli/pyproject.toml b/python/cli/pyproject.toml index eb4b4f9f..bf7dae2b 100644 --- a/python/cli/pyproject.toml +++ b/python/cli/pyproject.toml @@ -9,9 +9,9 @@ readme = "README.md" [tool.poetry.scripts] redact-secrets = "data_platform_workflows_cli.redact_secrets:main" -collect-snap-bases = "data_platform_workflows_cli.craft_tools.collect_bases:snap" -collect-rock-bases = "data_platform_workflows_cli.craft_tools.collect_bases:rock" -collect-charm-bases = "data_platform_workflows_cli.craft_tools.collect_bases:charm" +collect-snap-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:snap" +collect-rock-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:rock" +collect-charm-platforms = "data_platform_workflows_cli.craft_tools.collect_platforms:charm" release-snap = "data_platform_workflows_cli.craft_tools.release:snap" release-rock = "data_platform_workflows_cli.craft_tools.release:rock" release-charm = "data_platform_workflows_cli.craft_tools.release:charm" diff --git a/python/pytest_plugins/pytest_operator_cache/README.md b/python/pytest_plugins/pytest_operator_cache/README.md index 4060fca7..2175a180 100644 --- a/python/pytest_plugins/pytest_operator_cache/README.md +++ b/python/pytest_plugins/pytest_operator_cache/README.md @@ -1,3 +1,6 @@ +> [!WARNING] +> `pytest-operator-cache` is **deprecated**. Migration instructions: [deprecation_notice.md](deprecation_notice.md) + [pytest-operator](https://github.com/charmed-kubernetes/pytest-operator) plugin that overrides `ops_test.build_charm()` to return cached *.charm file instead of building new *.charm file. Usage: [integration_test_charm.md](../../../.github/workflows/integration_test_charm.md) \ No newline at end of file diff --git a/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md b/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md new file mode 100644 index 00000000..56139868 --- /dev/null +++ b/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md @@ -0,0 +1,14 @@ +# Deprecation notice + +`pytest-operator-cache` is deprecated & will be removed in a future release + +## Current behavior +With `pytest-operator-cache` installed, `ops_test.build_charm` (from [pytest-operator](https://github.com/charmed-kubernetes/pytest-operator)): +- On a local machine, builds the charm with charmcraft +- On CI (if `os.environ.get("CI") == "true"`), returns the path of an existing *.charm file—if the *.charm file was built on Ubuntu 22.04 + - (On CI, the *.charm files are built in a separate job [separate runner] from the integration tests) + +## Migration instructions +In integration tests, instead of calling `ops_test.build_charm`, assume the *.charm file exists and fail the test if the *.charm file is missing. The charm should be built outside of the test + +When running tests locally, if you would like the charm to be re-built each time the tests are run, consider using [charmcraftcache](https://github.com/canonical/charmcraftcache) (e.g. `ccc pack`) before the `pytest` command (e.g. in tox.ini). If you have multiple charms, `ccc pack` needs to be called once for each charm diff --git a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py index f7e1b26d..a62b5275 100644 --- a/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py +++ b/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py @@ -2,11 +2,16 @@ import pathlib import subprocess import typing - -import yaml +import warnings def pytest_configure(config): + warnings.warn( + # "\n::warning::" for https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message + "\n::warning::The `pytest-operator-cache` plugin is deprecated. Follow the migration instructions here: " + "https://github.com/canonical/data-platform-workflows/blob/v26.0.0/python/pytest_plugins/pytest_operator_cache/deprecation_notice.md", + DeprecationWarning, + ) if os.environ.get("CI") == "true": # Running in GitHub Actions; skip build step plugin = config.pluginmanager.get_plugin("pytest-operator") @@ -19,9 +24,7 @@ def pytest_configure(config): ) -async def build_charm( - self, charm_path: typing.Union[str, os.PathLike], bases_index: int = None -) -> pathlib.Path: +async def build_charm(self, charm_path: typing.Union[str, os.PathLike]) -> pathlib.Path: charm_path = pathlib.Path(charm_path) architecture = subprocess.run( ["dpkg", "--print-architecture"], @@ -30,24 +33,8 @@ async def build_charm( encoding="utf-8", ).stdout.strip() assert architecture in ("amd64", "arm64") - if bases_index is not None: - charmcraft_yaml = yaml.safe_load((charm_path / "charmcraft.yaml").read_text()) - assert charmcraft_yaml["type"] == "charm" - base = charmcraft_yaml["bases"][bases_index] - # Handle multiple base formats - # See https://discourse.charmhub.io/t/charmcraft-bases-provider-support/4713 - build_on = base.get("build-on", [base])[0] - version = build_on["channel"] - architectures = build_on.get("architectures", ["amd64"]) - assert ( - len(architectures) == 1 - ), f"Multiple architectures ({architectures}) in one (charmcraft.yaml) base not supported. Use one base per architecture" - assert ( - architectures[0] == architecture - ), f"Architecture for {bases_index=} ({architectures[0]}) does not match host architecture ({architecture})" - packed_charms = list(charm_path.glob(f"*{version}-{architecture}.charm")) - else: - packed_charms = list(charm_path.glob(f"*-{architecture}.charm")) + # 22.04 pin is temporary solution while multi-base integration testing not supported by data-platform-workflows + packed_charms = list(charm_path.glob(f"*ubuntu@22.04-{architecture}.charm")) if len(packed_charms) == 1: # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms # to begin with `./` or `/` to distinguish them from Charmhub charms. @@ -58,11 +45,11 @@ async def build_charm( # `pathlib.Path`.) return packed_charms[0].resolve(strict=True) elif len(packed_charms) > 1: - message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}." - if bases_index is None: - message += " Specify `bases_index`" - raise ValueError(message) + raise ValueError( + f"More than one matching .charm file found at {charm_path=} for {architecture=} and " + f"Ubuntu 22.04: {packed_charms}." + ) else: raise ValueError( - f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}" + f"Unable to find .charm file for {architecture=} and Ubuntu 22.04 at {charm_path=}" )