Deploy bundle and run UATs on self-hosted runners #286
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy bundle and run UATs on self-hosted runners | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| bundle-source: | |
| description: 'Either `--channel <channel_name>` or `--file <bundle_file>.yaml`' | |
| required: true | |
| uats-branch: | |
| description: Branch to run the UATs from e.g. main or track/1.7 | |
| required: false | |
| default: main | |
| microk8s-channel: | |
| description: Microk8s channel e.g. 1.25-strict/stable | |
| required: false | |
| default: "1.25-strict/stable" | |
| juju-channel: | |
| description: Juju channel e.g. 3.5/stable | |
| required: false | |
| default: "3.4/stable" | |
| workflow_call: | |
| inputs: | |
| bundle-source: | |
| description: 'Either `--channel <channel_name>` or `--file <bundle_file>.yaml`' | |
| type: string | |
| required: true | |
| uats-branch: | |
| description: Branch to run the UATs from e.g. main or track/1.7 | |
| required: false | |
| type: string | |
| default: main | |
| microk8s-channel: | |
| description: Microk8s channel e.g. 1.25-strict/stable | |
| required: false | |
| type: string | |
| default: "1.25-strict/stable" | |
| juju-channel: | |
| description: Juju channel e.g. 3.5/stable | |
| required: false | |
| type: string | |
| default: "3.4/stable" | |
| jobs: | |
| test-bundle: | |
| runs-on: [self-hosted, linux, X64, two-xlarge] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Parse and enable DNS server | |
| id: dns-name | |
| run: | | |
| dns_server=$(grep -oPm1 'Current DNS Server: \K[^\s]+' <<< "$(resolvectl status)") | |
| echo "Using following DNS Server: $dns_server" | |
| echo "MY_ADDONS=hostpath-storage ingress dns:$dns_server rbac registry metallb:'10.64.140.43-10.64.140.49,192.168.0.105-192.168.0.111'" >> $GITHUB_OUTPUT | |
| # This is needed in order to configure internet connection for self-hosted runner. | |
| # Source: https://discourse.canonical.com/t/introducing-aproxy-a-transparent-proxy-for-github-self-hosted-runners/2566 | |
| - name: Setup aproxy | |
| run: | | |
| sudo snap install aproxy --edge | |
| sudo snap set aproxy proxy=squid.internal:3128 | |
| sudo nft -f - << EOF | |
| define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') | |
| define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } | |
| table ip aproxy | |
| flush table ip aproxy | |
| table ip aproxy { | |
| chain prerouting { | |
| type nat hook prerouting priority dstnat; policy accept; | |
| ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:8443 | |
| } | |
| chain output { | |
| type nat hook output priority -100; policy accept; | |
| ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:8443 | |
| } | |
| } | |
| EOF | |
| - name: Install tox | |
| run: | | |
| pip install tox | |
| - name: Setup operator environment | |
| uses: charmed-kubernetes/actions-operator@main | |
| with: | |
| provider: microk8s | |
| channel: ${{ inputs.microk8s-channel }} | |
| juju-channel: ${{ inputs.juju-channel }} | |
| charmcraft-channel: latest/candidate | |
| microk8s-addons: ${{ steps.dns-name.outputs.MY_ADDONS }} | |
| - name: Wait for microk8s to be ready and configure .kube/config | |
| run: | | |
| sudo microk8s status --wait-ready --timeout 150 | |
| sg microk8s -c "mkdir -p ~/.kube" | |
| sudo microk8s config > ~/.kube/config | |
| - name: Show all pods status | |
| run: | | |
| sudo microk8s kubectl get pods --all-namespaces | |
| - name: Increase file system limits | |
| run: | | |
| sudo sysctl fs.inotify.max_user_instances=1280 | |
| sudo sysctl fs.inotify.max_user_watches=655360 | |
| - name: Configure Juju model | |
| run: | | |
| sg microk8s -c "juju add-model kubeflow --config default-series=focal --config automatically-retry-hooks=true" | |
| sg microk8s -c "juju model-config" | |
| sg microk8s -c "juju status" | |
| - name: Get release from bundle-source input | |
| id: get-release-from-bundle-source | |
| run: python scripts/get_release_from_bundle_source.py "${{ inputs.bundle-source }}" | |
| - name: Get bundle test path for ${{ steps.get-release-from-bundle-source.outputs.release }} | |
| id: bundle-test-path | |
| run: python scripts/get_bundle_test_path.py ${{ steps.get-release-from-bundle-source.outputs.release }} | |
| - name: Run bundle tests | |
| run: | | |
| eval "$(pyenv init -)" | |
| export BUNDLE_TEST_PATH=${{ steps.bundle-test-path.outputs.bundle_test_path }} | |
| export GH_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| sg microk8s -c "tox -e full_bundle_tests -- ${{ inputs.bundle-source }}" | |
| - name: Checkout Kubeflow UATs | |
| run: | | |
| git clone https://github.com/canonical/charmed-kubeflow-uats.git ~/charmed-kubeflow-uats | |
| cd ~/charmed-kubeflow-uats | |
| git checkout ${{ inputs.uats-branch }} | |
| - name: Run UATs | |
| run: | | |
| eval "$(pyenv init -)" | |
| sg microk8s -c "tox -c ~/charmed-kubeflow-uats/ -e kubeflow-local" | |
| - name: Save debug artifacts | |
| uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main | |
| if: always() | |
| - name: Dump Aproxy logs on failure | |
| if: failure() || cancelled() | |
| run: sudo snap logs aproxy.aproxy -n=all | |
| - name: Run connectivity check | |
| if: always() | |
| run: | | |
| sg microk8s -c "curl --max-time 10 --connect-timeout 10 http://10.64.140.43.nip.io" | |
| - name: Dump Juju/k8s logs on failure | |
| if: failure() || cancelled() | |
| run: | | |
| sg microk8s -c "juju status" | |
| echo "Dumping k8s logs" | |
| sudo microk8s kubectl get all --all-namespaces | |
| - name: Dump juju debug-log on failure | |
| if: failure() || cancelled() | |
| run: | | |
| sg microk8s -c "juju debug-log --replay --no-tail" | |
| - name: Descript all pods | |
| if: failure() || cancelled() | |
| run: | | |
| sudo microk8s kubectl describe pods --all-namespaces | |
| - name: Get logs from failed pods | |
| if: failure() || cancelled() | |
| run: | | |
| POD_LIST=$(sudo microk8s kubectl get pods --all-namespaces -o 'custom-columns=NAME:.metadata.name,NAMESPACE:.metadata.namespace,CONTAINERS:.status.containerStatuses[*].ready' | awk '$3 == "false" {print $1,$2}') | |
| if [ -n "$POD_LIST" ]; then | |
| echo "Actual Logs" | |
| while read -r POD NAMESPACE; do | |
| echo "\n\n\nPod: $POD" | |
| sudo microk8s kubectl logs -n $NAMESPACE $POD | |
| done <<< "$POD_LIST" | |
| fi |