From d20c4f639b6e06bb6f92b6f362452df4fb0b17b7 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Wed, 18 Dec 2024 11:41:35 +0200 Subject: [PATCH] Deduplicate github actions (#892) * Deduplicate github actions We have multiple github actions that run e2e tests and share a significant amount of logic. We'll add reusable actions, making the workflows much easier to maintain. * Fix flaky microk8s test As part of the test cleanup, we're removing the k8s snap, ensuring that its services and mounts go away. One of the tests installs microk8s, which interferes with the k8s snap cleanup assertions. We'll fix this flaky test by removing the microk8s snap. * Fix flaky ingress test get_external_service_ip returns an empty string, however the test asserts that the ip is not None and proceeds with the curl: 2024-12-12 11:28:46 DEBUG Execute command ['curl', '', '-H', 'Host: foo.bar.com'] in instance k8s-integration-530bc4-37 We'll update the assertion and catch empty strings as well. At the same time, we'll increase the timeouts to reduce test flakiness. * Merge nightly test and cron job The nightly job is also a cron job that executes daily, so it makes sense to merge those two workflows. * Fix nightly job tag * Pass test flavor * Include all namespaces in inspection reports The moonray job is failing, however we only have logs from the "default" and "kube-system" namespaces. This change will collect logs from all k8s namespaces. * Apply flavor patches before running the tests We'll need to apply the strict/moonray patches not only when building the snap, but also when running the tests. * Skip broken test test_containerd_path_cleanup_on_failed_init holds an open port and expects the bootstrap to fail, however that won't be the case when using the lxd harness. We'll skip this test for now. * Revert "Include all namespaces in inspection reports" This reverts commit 5020f3978648ad04cd989cf494ec77e938c4bc2a. * Address PR feedback * cover 1.32 as part of the nightly tests * get go version from go.mod * update step names * add some TODOs * make lxd channel configurable * bump ubuntu versions * add get-e2e-tags dependencies --- .github/actions/download-k8s-snap/action.yaml | 61 ++++++ .github/actions/install-lxd/action.yaml | 28 +++ .github/workflows/build-snap.yaml | 54 +++++ .github/workflows/cron-jobs.yaml | 126 ------------ .github/workflows/e2e-tests.yaml | 84 ++++++++ .github/workflows/go.yaml | 20 +- .github/workflows/integration-informing.yaml | 104 ---------- .github/workflows/integration.yaml | 187 ------------------ .github/workflows/lint_and_integration.yaml | 140 +++++++++++++ .github/workflows/nightly-test.yaml | 136 ++++++++----- .github/workflows/python.yaml | 37 ---- .github/workflows/security-scan.yaml | 81 ++++++++ .github/workflows/update-branches.yaml | 2 +- k8s/scripts/inspect.sh | 1 + tests/integration/tests/test_bootstrap.py | 2 + tests/integration/tests/test_cleanup.py | 1 + tests/integration/tests/test_ingress.py | 8 +- 17 files changed, 547 insertions(+), 525 deletions(-) create mode 100644 .github/actions/download-k8s-snap/action.yaml create mode 100644 .github/actions/install-lxd/action.yaml create mode 100644 .github/workflows/build-snap.yaml delete mode 100644 .github/workflows/cron-jobs.yaml create mode 100644 .github/workflows/e2e-tests.yaml delete mode 100644 .github/workflows/integration-informing.yaml delete mode 100644 .github/workflows/integration.yaml create mode 100644 .github/workflows/lint_and_integration.yaml delete mode 100644 .github/workflows/python.yaml create mode 100644 .github/workflows/security-scan.yaml diff --git a/.github/actions/download-k8s-snap/action.yaml b/.github/actions/download-k8s-snap/action.yaml new file mode 100644 index 000000000..75c6a12cf --- /dev/null +++ b/.github/actions/download-k8s-snap/action.yaml @@ -0,0 +1,61 @@ +name: Download k8s-snap + +inputs: + # Download k8s-snap using either a GH action artifact or a snap channel. + artifact: + description: The name of a GH action artifact. + type: string + channel: + description: k8s snap channel. + type: string + +outputs: + snap-path: + description: The *.snap destination path. + value: ${{ steps.retrieve-path.outputs.snap-path }} + +runs: + using: "composite" + steps: + - name: Fail if neither artifact nor channel are specified + if: inputs.artifact == '' && inputs.channel == '' + shell: bash + run: | + echo "No k8s-snap artifact or channel specified..." + exit 1 + - name: Fail if both artifact and channel are specified + if: inputs.artifact != '' && inputs.channel != '' + shell: bash + run: | + echo "Received snap artifact AND snap channel." + exit 1 + + - name: Download snap artifact + if: inputs.artifact != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact }} + path: ${{ github.workspace }} + + - name: Download snap channel + if: inputs.channel != '' + shell: bash + run: | + snap download k8s --channel=${{ inputs.channel }} --basename k8s + full_path=`realpath k8s.snap` + echo "Downloaded snap: $full_path" + ls -lh $full_path + + - name: Retrieve resulting snap path + shell: bash + id: retrieve-path + run: | + if [[ -n "${{ inputs.artifact }}" ]]; then + snap_path="${{ github.workspace }}/${{ inputs.artifact }}" + else + snap_path="${{ github.workspace }}/k8s.snap" + fi + echo "snap-path=$snap_path" >> "$GITHUB_OUTPUT" + + echo "Output snap: $snap_path" + ls -lh $snap_path diff --git a/.github/actions/install-lxd/action.yaml b/.github/actions/install-lxd/action.yaml new file mode 100644 index 000000000..3e90cd23b --- /dev/null +++ b/.github/actions/install-lxd/action.yaml @@ -0,0 +1,28 @@ +name: Install lxd + +inputs: + channel: + description: lxd snap channel. + type: string + default: "5.21/stable" + +runs: + using: "composite" + steps: + - name: Install lxd snap + shell: bash + run: | + sudo snap refresh lxd --channel ${{ inputs.channel }} + - name: Initialize lxd + shell: bash + run: | + sudo lxd init --auto + sudo usermod --append --groups lxd $USER + sg lxd -c 'lxc version' + # Docker sets iptables rules that interfere with LXD. + # https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker + - name: Apply Docker iptables workaround + shell: bash + run: | + sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT + sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT diff --git a/.github/workflows/build-snap.yaml b/.github/workflows/build-snap.yaml new file mode 100644 index 000000000..fbc39d01c --- /dev/null +++ b/.github/workflows/build-snap.yaml @@ -0,0 +1,54 @@ +name: Build k8s-snap + +permissions: + contents: read + +on: + workflow_call: + inputs: + flavor: + description: k8s-snap flavor (e.g. moonray or strict) + type: string + outputs: + snap-artifact: + description: Name of the uploaded snap artifact + value: ${{ jobs.build-snap.outputs.snap-artifact }} + +jobs: + build-snap: + name: Build snap + runs-on: ubuntu-latest + outputs: + snap-artifact: ${{ steps.build.outputs.snap-artifact }} + steps: + - name: Checking out repo + uses: actions/checkout@v4 + - name: Apply flavor-specific patches + if: inputs.flavor != '' + run: | + ./build-scripts/patches/${{ inputs.flavor }}/apply + - name: Install lxd + uses: ./.github/actions/install-lxd + - name: Install snapcraft + run: | + sudo snap install snapcraft --classic + - name: Build k8s snap + id: build + env: + flavor: ${{ inputs.flavor }} + run: | + if [[ -n "$flavor" ]]; then + out_snap=k8s-$flavor.snap + else + out_snap=k8s.snap + fi + + sg lxd -c 'snapcraft --use-lxd' + mv k8s_*.snap $out_snap + + echo "snap-artifact=$out_snap" >> "$GITHUB_OUTPUT" + - name: Upload k8s snap + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.build.outputs.snap-artifact }} + path: ${{ steps.build.outputs.snap-artifact }} diff --git a/.github/workflows/cron-jobs.yaml b/.github/workflows/cron-jobs.yaml deleted file mode 100644 index 6fcc85631..000000000 --- a/.github/workflows/cron-jobs.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: Security and quality nightly scan - -on: - schedule: - - cron: '0 10 * * *' - -permissions: - contents: read - -jobs: - TICS: - permissions: - contents: read - runs-on: ubuntu-latest - strategy: - matrix: - include: - # Latest branches - - { branch: main } - - steps: - - name: Checking out repo - uses: actions/checkout@v4 - with: - ref: ${{matrix.branch}} - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: "1.22" - - name: go mod download - working-directory: src/k8s - run: go mod download - - name: TICS scan - run: | - export TICSAUTHTOKEN=${{ secrets.TICSAUTHTOKEN }} - - set -x - # Install python dependencies - pip install -r tests/integration/requirements-test.txt - pip install -r tests/integration/requirements-dev.txt - - cd src/k8s - - # TICS requires us to have the test results in cobertura xml format under the - # directory use below - sudo make go.unit - go install github.com/boumenot/gocover-cobertura@latest - gocover-cobertura < coverage.txt > coverage.xml - mkdir .coverage - mv ./coverage.xml ./.coverage/ - - # Install the TICS and staticcheck - go install honnef.co/go/tools/cmd/staticcheck@v0.5.1 - . <(curl --silent --show-error 'https://canonical.tiobe.com/tiobeweb/TICS/api/public/v1/fapi/installtics/Script?cfg=default&platform=linux&url=https://canonical.tiobe.com/tiobeweb/TICS/') - - # We need to have our project built - # We load the dqlite libs here instead of doing through make because TICS - # will try to build parts of the project itself - sudo add-apt-repository -y ppa:dqlite/dev - sudo apt install dqlite-tools-v2 libdqlite1.17-dev - sudo make clean - go build -a ./... - - TICSQServer -project k8s-snap -tmpdir /tmp/tics -branchdir $HOME/work/k8s-snap/k8s-snap/ - - Trivy: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - runs-on: ubuntu-latest - strategy: - matrix: - include: - # Latest branches - - { branch: main, channel: latest/edge } - # Stable branches - # Add branches to test here - - { branch: release-1.30, channel: 1.30-classic/edge } - - { branch: release-1.31, channel: 1.31-classic/edge } - - steps: - - name: Checking out repo - uses: actions/checkout@v4 - with: - ref: ${{matrix.branch}} - - name: Setup Trivy vulnerability scanner - run: | - mkdir -p sarifs - VER=$(curl --silent -qI https://github.com/aquasecurity/trivy/releases/latest | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}'); - wget https://github.com/aquasecurity/trivy/releases/download/${VER}/trivy_${VER#v}_Linux-64bit.tar.gz - tar -zxvf ./trivy_${VER#v}_Linux-64bit.tar.gz - - name: Run Trivy vulnerability scanner in repo mode - uses: aquasecurity/trivy-action@master - with: - scan-type: "fs" - ignore-unfixed: true - format: "sarif" - output: "trivy-k8s-repo-scan--results.sarif" - severity: "MEDIUM,HIGH,CRITICAL" - env: - TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db" - - name: Gather Trivy repo scan results - run: | - cp trivy-k8s-repo-scan--results.sarif ./sarifs/ - - name: Run Trivy vulnerability scanner on the snap - run: | - snap download k8s --channel ${{ matrix.channel }} - mv ./k8s*.snap ./k8s.snap - unsquashfs k8s.snap - for var in $(env | grep -o '^TRIVY_[^=]*'); do - unset "$var" - done - ./trivy --db-repository public.ecr.aws/aquasecurity/trivy-db rootfs ./squashfs-root/ --format sarif > sarifs/snap.sarif - - name: Get HEAD sha - run: | - SHA="$(git rev-parse HEAD)" - echo "head_sha=$SHA" >> "$GITHUB_ENV" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "sarifs" - sha: ${{ env.head_sha }} - ref: refs/heads/${{matrix.branch}} diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml new file mode 100644 index 000000000..4e2aa086a --- /dev/null +++ b/.github/workflows/e2e-tests.yaml @@ -0,0 +1,84 @@ +name: Run k8s-snap e2e tests + +permissions: + contents: read + +on: + workflow_call: + inputs: + arch: + description: Job runner architecture (amd64 or arm64) + default: amd64 + type: string + os: + description: LXD image to use when running e2e tests + default: ubuntu:24.04 + type: string + # Download k8s-snap using either a GH action artifact or a snap channel. + artifact: + description: The name of a GH action artifact. + type: string + channel: + description: k8s snap channel. + type: string + test-tags: + description: Integration test filter tags (e.g. pull_request, up_to_weekly) + default: pull_request + type: string + flavor: + description: Test flavor (e.g. moonray or strict) + default: "" + type: string + +jobs: + test-integration: + name: Integration Test ${{ inputs.os }} ${{ inputs.arch }} ${{ inputs.artifact }} + runs-on: ${{ inputs.arch == 'arm64' && 'self-hosted-linux-arm64-jammy-large' || 'self-hosted-linux-amd64-jammy-large' }} + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Apply patches + if: inputs.flavor != '' + run: | + ./build-scripts/patches/${{ inputs.flavor }}/apply + - name: Download k8s-snap + id: download-snap + uses: ./.github/actions/download-k8s-snap + with: + channel: ${{ inputs.channel }} + artifact: ${{ inputs.artifact }} + - name: Install lxd + uses: ./.github/actions/install-lxd + - name: Install tox + run: pip install tox + - name: Run e2e tests + env: + TEST_SNAP: ${{ steps.download-snap.outputs.snap-path }} + TEST_SUBSTRATE: lxd + TEST_LXD_IMAGE: ${{ inputs.os }} + TEST_FLAVOR: ${{ inputs.flavor }} + TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports + # Test the latest (up to) 6 releases for the flavour + # TODO(ben): upgrade nightly to run all flavours + TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic" + # Upgrading from 1.30 is not supported. + TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31" + TEST_STRICT_INTERFACE_CHANNELS: "recent 6 strict" + TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}, {"name": "rocks.canonical.com", "port": 5002, "remote": "https://rocks.canonical.com/cdk"}]' + run: | + cd tests/integration && sg lxd -c "tox -e integration -- --tags ${{ inputs.test-tags }}" + - name: Prepare inspection reports + if: failure() + run: | + tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports + echo "artifact_name=inspection-reports-${{ inputs.os }}" | sed 's/:/-/g' >> $GITHUB_ENV + - name: Upload inspection report artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.artifact_name }} + path: ${{ github.workspace }}/inspection-reports.tar.gz diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index c5b7ca1ca..dc2a9bc87 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -1,19 +1,7 @@ -name: Go +name: Go lint and unit tests on: - push: - paths-ignore: - - 'docs/**' - branches: - - main - - autoupdate/strict - - autoupdate/moonray - - 'release-[0-9]+.[0-9]+' - - 'autoupdate/release-[0-9]+.[0-9]+-strict' - - 'autoupdate/sync/**' - pull_request: - paths-ignore: - - 'docs/**' + workflow_call: permissions: contents: read @@ -34,7 +22,7 @@ jobs: - name: Install Go uses: actions/setup-go@v5 with: - go-version: "1.23" + go-version-file: './src/k8s/go.mod' - name: go mod download working-directory: src/k8s @@ -92,7 +80,7 @@ jobs: - name: Install Go uses: actions/setup-go@v5 with: - go-version: "1.23" + go-version-file: './src/k8s/go.mod' - name: go mod download working-directory: src/k8s diff --git a/.github/workflows/integration-informing.yaml b/.github/workflows/integration-informing.yaml deleted file mode 100644 index d569a7c66..000000000 --- a/.github/workflows/integration-informing.yaml +++ /dev/null @@ -1,104 +0,0 @@ -name: Informing Integration Tests - -on: - push: - paths-ignore: - - 'docs/**' - branches: - - main - - 'release-[0-9]+.[0-9]+' - - 'autoupdate/sync/**' - pull_request: - paths-ignore: - - 'docs/**' - -permissions: - contents: read - -jobs: - build: - name: Build ${{ matrix.patch }} - runs-on: ubuntu-20.04 - strategy: - matrix: - patch: ["moonray"] - fail-fast: false - steps: - - name: Checking out repo - uses: actions/checkout@v4 - - name: Install lxd - run: | - sudo snap refresh lxd --channel 5.21/stable - sudo lxd init --auto - sudo usermod --append --groups lxd $USER - sg lxd -c 'lxc version' - - name: Install snapcraft - run: | - sudo snap install snapcraft --classic - - name: Apply ${{ matrix.patch }} patch - run: | - ./build-scripts/patches/${{ matrix.patch }}/apply - - name: Build snap - run: | - sg lxd -c 'snapcraft --use-lxd' - mv k8s_*.snap k8s-${{ matrix.patch }}.snap - - name: Uploading snap - uses: actions/upload-artifact@v4 - with: - name: k8s-${{ matrix.patch }}.snap - path: k8s-${{ matrix.patch }}.snap - - test-integration: - needs: [ build ] - name: Test ${{ matrix.patch }} ${{ matrix.os }} - strategy: - matrix: - os: ["ubuntu:20.04"] - patch: ["moonray"] - fail-fast: false - runs-on: ["self-hosted", "Linux", "AMD64", "jammy", "large"] - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Install tox - run: pip install tox - - name: Install lxd - run: | - sudo snap refresh lxd --channel 5.21/stable - sudo lxd init --auto - sudo usermod --append --groups lxd $USER - sg lxd -c 'lxc version' - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - name: Download snap - uses: actions/download-artifact@v4 - with: - name: k8s-${{ matrix.patch }}.snap - path: ${{ github.workspace }}/build - - name: Apply ${{ matrix.patch }} patch - run: | - ./build-scripts/patches/${{ matrix.patch }}/apply - - name: Run end to end tests - env: - TEST_SNAP: ${{ github.workspace }}/build/k8s-${{ matrix.patch }}.snap - TEST_SUBSTRATE: lxd - TEST_LXD_IMAGE: ${{ matrix.os }} - TEST_FLAVOR: ${{ matrix.patch }} - TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports - run: | - cd tests/integration && sg lxd -c 'tox -e integration -- --tags pull_request' - - name: Prepare inspection reports - if: failure() - run: | - tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports - echo "artifact_name=inspection-reports-${{ matrix.os }}-${{ matrix.patch }}" | sed 's/:/-/g' >> $GITHUB_ENV - - name: Upload inspection report artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: ${{ env.artifact_name }} - path: ${{ github.workspace }}/inspection-reports.tar.gz diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml deleted file mode 100644 index 13857352e..000000000 --- a/.github/workflows/integration.yaml +++ /dev/null @@ -1,187 +0,0 @@ -name: Integration Tests - -on: - push: - paths-ignore: - - 'docs/**' - branches: - - main - - autoupdate/strict - - autoupdate/moonray - - 'release-[0-9]+.[0-9]+' - - 'autoupdate/release-[0-9]+.[0-9]+-strict' - - 'autoupdate/sync/**' - pull_request: - paths-ignore: - - 'docs/**' - -permissions: - contents: read - -jobs: - build: - name: Build - runs-on: ubuntu-20.04 - - steps: - - name: Checking out repo - uses: actions/checkout@v4 - - name: Install lxd - run: | - sudo snap refresh lxd --channel 5.21/stable - sudo lxd init --auto - sudo usermod --append --groups lxd $USER - sg lxd -c 'lxc version' - - name: Install snapcraft - run: | - sudo snap install snapcraft --classic - - name: Build snap - run: | - sg lxd -c 'snapcraft --use-lxd' - mv k8s_*.snap k8s.snap - - name: Uploading snap - uses: actions/upload-artifact@v4 - with: - name: k8s.snap - path: k8s.snap - - test-branches: - name: Test Branch Management - runs-on: ubuntu-20.04 - steps: - - name: Check out code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Install tox - run: pip install tox - - name: Run branch_management tests - run: | - tox -c tests/branch_management -e test - - test-integration: - name: Test ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] - runs-on: ["self-hosted", "Linux", "AMD64", "jammy", "large"] - needs: build - - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Install tox - run: pip install tox - - name: Install lxd - run: | - sudo snap refresh lxd --channel 5.21/stable - sudo lxd init --auto - sudo usermod --append --groups lxd $USER - sg lxd -c 'lxc version' - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - name: Download snap - uses: actions/download-artifact@v4 - with: - name: k8s.snap - path: ${{ github.workspace }}/build - - name: Run end to end tests - env: - TEST_SNAP: ${{ github.workspace }}/build/k8s.snap - TEST_SUBSTRATE: lxd - TEST_LXD_IMAGE: ${{ matrix.os }} - TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports - # Test the latest (up to) 6 releases for the flavour - # TODO(ben): upgrade nightly to run all flavours - TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic" - # Upgrading from 1.30 is not supported. - TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31" - TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}, {"name": "rocks.canonical.com", "port": 5002, "remote": "https://rocks.canonical.com/cdk"}]' - run: | - tags="pull_request" - # Run all tests if there are test changes. In case of a PR, we'll - # get a merge commit that includes all changes. - if git diff HEAD HEAD~1 --name-only | grep "tests/"; then - tags="up_to_weekly" - fi - # Run all tests on backports. - if echo ${{ github.base_ref }} | grep "release-"; then - tags="up_to_weekly" - fi - cd tests/integration && sg lxd -c "tox -e integration -- --tags $tags" - - name: Prepare inspection reports - if: failure() - run: | - tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports - echo "artifact_name=inspection-reports-${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV - - name: Upload inspection report artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: ${{ env.artifact_name }} - path: ${{ github.workspace }}/inspection-reports.tar.gz - - security-scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - name: Security scan - runs-on: ubuntu-20.04 - needs: build - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - # We run into rate limiting issues if we don't authenticate - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Checking out repo - uses: actions/checkout@v4 - - name: Fetch snap - uses: actions/download-artifact@v4 - with: - name: k8s.snap - path: build - - name: Setup Trivy vulnerability scanner - run: | - mkdir -p manual-trivy/sarifs - pushd manual-trivy - VER=$(curl --silent -qI https://github.com/aquasecurity/trivy/releases/latest | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}'); - wget https://github.com/aquasecurity/trivy/releases/download/${VER}/trivy_${VER#v}_Linux-64bit.tar.gz - tar -zxvf ./trivy_${VER#v}_Linux-64bit.tar.gz - popd - - name: Run Trivy vulnerability scanner in repo mode - uses: aquasecurity/trivy-action@master - with: - scan-type: "fs" - ignore-unfixed: true - format: "sarif" - output: "trivy-k8s-repo-scan--results.sarif" - severity: "MEDIUM,HIGH,CRITICAL" - env: - TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db" - - name: Gather Trivy repo scan results - run: | - cp trivy-k8s-repo-scan--results.sarif ./manual-trivy/sarifs/ - - name: Run Trivy vulnerability scanner on the snap - run: | - for var in $(env | grep -o '^TRIVY_[^=]*'); do - unset "$var" - done - cp build/k8s.snap . - unsquashfs k8s.snap - ./manual-trivy/trivy --db-repository public.ecr.aws/aquasecurity/trivy-db rootfs ./squashfs-root/ --format sarif > ./manual-trivy/sarifs/snap.sarif - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: "./manual-trivy/sarifs" diff --git a/.github/workflows/lint_and_integration.yaml b/.github/workflows/lint_and_integration.yaml new file mode 100644 index 000000000..6e92af48b --- /dev/null +++ b/.github/workflows/lint_and_integration.yaml @@ -0,0 +1,140 @@ +name: Lint and integration tests + +on: + push: + paths-ignore: + - 'docs/**' + branches: + - main + - autoupdate/strict + - autoupdate/moonray + - 'release-[0-9]+.[0-9]+' + - 'autoupdate/release-[0-9]+.[0-9]+-strict' + pull_request: + paths-ignore: + - 'docs/**' + +permissions: + contents: read + +jobs: + build-snap: + name: Build k8s-snap ${{ matrix.patch }} + uses: ./.github/workflows/build-snap.yaml + strategy: + matrix: + patch: ["", "moonray"] + with: + flavor: ${{ matrix.patch }} + + test-branches: + name: Test Branch Management + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.8' + - name: Install tox + run: pip install tox + - name: Run branch_management tests + run: | + tox -c tests/branch_management -e test + + python-lint: + name: Python lint + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install tox + run: pip install tox + - name: Lint + run: | + cd tests/integration && tox -e lint + + go-lint-and-unit: + name: Go lint and unit tests + uses: ./.github/workflows/go.yaml + permissions: + contents: read # for actions/checkout to fetch code + pull-requests: write # for marocchino/sticky-pull-request-comment to create or update PR comment + checks: write # for golangci/golangci-lint-action to checks to allow the action to annotate code in the PR. + + get-e2e-tags: + name: Get e2e test tags + runs-on: ubuntu-latest + outputs: + test-tags: ${{ steps.get-e2e-tags.outputs.test-tags }} + needs: [build-snap, go-lint-and-unit, python-lint] + steps: + - name: Checking out repo + uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Get e2e test tags + id: get-e2e-tags + run: | + tags="pull_request" + if ${{ github.event_name == 'pull_request' }}; then + # Run all tests if there are test changes. In case of a PR, we'll + # get a merge commit that includes all changes. + if git diff HEAD HEAD~1 --name-only | grep "tests/"; then + tags="up_to_weekly" + fi + # Run all tests on backports. + if echo ${{ github.base_ref }} | grep "release-"; then + tags="up_to_weekly" + fi + fi + + echo "test-tags=$tags" >> "$GITHUB_OUTPUT" + + test-integration: + name: Test ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] + needs: [build-snap, get-e2e-tags, go-lint-and-unit, python-lint] + uses: ./.github/workflows/e2e-tests.yaml + with: + arch: amd64 + os: ${{ matrix.os }} + test-tags: ${{ needs.get-e2e-tags.outputs.test-tags}} + artifact: k8s.snap + + test-integration-informing: + name: Test informing ${{ matrix.os }} ${{ matrix.patch }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu:24.04"] + patch: ["moonray"] + needs: [build-snap, get-e2e-tags, go-lint-and-unit, python-lint] + if: success() && github.event_name == 'pull_request' + uses: ./.github/workflows/e2e-tests.yaml + with: + arch: amd64 + os: ${{ matrix.os }} + test-tags: ${{ needs.get-e2e-tags.outputs.test-tags}} + artifact: k8s-${{ matrix.patch }}.snap + flavor: ${{ matrix.patch }} + + security-scan: + name: Security scan + needs: build-snap + uses: ./.github/workflows/security-scan.yaml + with: + artifact: ${{ needs.build-snap.outputs.snap-artifact}} + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml index 98089d932..9b1c65ebc 100644 --- a/.github/workflows/nightly-test.yaml +++ b/.github/workflows/nightly-test.yaml @@ -1,8 +1,13 @@ -name: Nightly Latest/Edge Tests +name: Nightly Tests on: schedule: - cron: "0 0 * * *" # Runs every midnight + pull_request: + paths: + - .github/workflows/nightly-test.yaml + - .github/workflows/e2e-tests.yaml + - .github/workflows/security-scan.yaml permissions: contents: read @@ -14,58 +19,89 @@ jobs: matrix: os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] arch: ["amd64", "arm64"] - release: ["latest/edge"] - fail-fast: false # TODO: remove once arm64 works + channel: ["latest/edge"] + fail-fast: false # TODO: remove once we no longer have flaky tests. + uses: ./.github/workflows/e2e-tests.yaml + with: + arch: ${{ matrix.arch }} + os: ${{ matrix.os }} + channel: ${{ matrix.channel }} + test-tags: 'up_to_nightly' - runs-on: ${{ matrix.arch == 'arm64' && 'self-hosted-linux-arm64-jammy-large' || 'self-hosted-linux-amd64-jammy-large' }} + Trivy: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + strategy: + matrix: + include: + # Latest branches + - { branch: main, channel: latest/edge } + # Stable branches + # Add branches to test here + # TODO: automatically retrieve the list of channels. + - { branch: release-1.30, channel: 1.30-classic/edge } + - { branch: release-1.31, channel: 1.31-classic/edge } + - { branch: release-1.32, channel: 1.32-classic/edge } + uses: ./.github/workflows/security-scan.yaml + with: + channel: ${{ matrix.channel }} + checkout-ref: ${{ matrix.branch }} + TICS: + permissions: + contents: read + runs-on: ubuntu-latest + strategy: + matrix: + include: + # Latest branches + - { branch: main } steps: - name: Checking out repo uses: actions/checkout@v4 - - name: Install lxd and tox - run: | - sudo apt update - sudo apt install -y tox - sudo snap refresh lxd --channel 5.21/stable - sudo lxd init --auto - sudo usermod --append --groups lxd $USER - sg lxd -c 'lxc version' - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - - name: Create build directory - run: mkdir -p build - - name: Install ${{ matrix.release }} k8s snap - run: | - cd build - snap download k8s --channel=${{ matrix.release }} --basename k8s - - name: Run end to end tests # tox path needs to be specified for arm64 - env: - TEST_SNAP: ${{ github.workspace }}/build/k8s.snap - TEST_SUBSTRATE: lxd - TEST_LXD_IMAGE: ${{ matrix.os }} - TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports - # Test the latest (up to) 6 releases for the flavour - # TODO(ben): upgrade nightly to run all flavours - TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic" - # Upgrading from 1.30 is not supported. - TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31" - TEST_STRICT_INTERFACE_CHANNELS: "recent 6 strict" - TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}, {"name": "rocks.canonical.com", "port": 5002, "remote": "https://rocks.canonical.com/cdk"}]' - run: | - export PATH="/home/runner/.local/bin:$PATH" - cd tests/integration && sg lxd -c 'tox -vve integration -- --tags up_to_nightly ' - - name: Prepare inspection reports - if: failure() - run: | - tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports - echo "artifact_name=inspection-reports-${{ matrix.os }}-${{ matrix.arch }}" | sed 's/:/-/g' >> $GITHUB_ENV - - name: Upload inspection report artifact - if: failure() - uses: actions/upload-artifact@v4 with: - name: ${{ env.artifact_name }} - path: ${{ github.workspace }}/inspection-reports.tar.gz - - name: Tmate debugging session - if: ${{ failure() && github.event_name == 'pull_request' }} - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 10 + ref: ${{matrix.branch}} + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: './src/k8s/go.mod' + - name: go mod download + working-directory: src/k8s + run: go mod download + - name: TICS scan + # TODO: move the following to a script. + run: | + export TICSAUTHTOKEN=${{ secrets.TICSAUTHTOKEN }} + + set -x + # Install python dependencies + pip install -r tests/integration/requirements-test.txt + pip install -r tests/integration/requirements-dev.txt + + cd src/k8s + + # TICS requires us to have the test results in cobertura xml format under the + # directory use below + sudo make go.unit + go install github.com/boumenot/gocover-cobertura@latest + gocover-cobertura < coverage.txt > coverage.xml + mkdir .coverage + mv ./coverage.xml ./.coverage/ + + # Install the TICS and staticcheck + go install honnef.co/go/tools/cmd/staticcheck@v0.5.1 + . <(curl --silent --show-error 'https://canonical.tiobe.com/tiobeweb/TICS/api/public/v1/fapi/installtics/Script?cfg=default&platform=linux&url=https://canonical.tiobe.com/tiobeweb/TICS/') + + # We need to have our project built + # We load the dqlite libs here instead of doing through make because TICS + # will try to build parts of the project itself + sudo add-apt-repository -y ppa:dqlite/dev + sudo apt install dqlite-tools-v2 libdqlite1.17-dev + sudo make clean + go build -a ./... + + TICSQServer -project k8s-snap -tmpdir /tmp/tics -branchdir $HOME/work/k8s-snap/k8s-snap/ diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml deleted file mode 100644 index c8cb71c85..000000000 --- a/.github/workflows/python.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: Python - -on: - push: - paths-ignore: - - 'docs/**' - branches: - - main - - autoupdate/strict - - autoupdate/moonray - - 'release-[0-9]+.[0-9]+' - - 'autoupdate/release-[0-9]+.[0-9]+-strict' - - 'autoupdate/sync/**' - pull_request: - paths-ignore: - - 'docs/**' - -permissions: - contents: read - -jobs: - lint: - name: Lint - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Install tox - run: pip install tox - - name: Lint - run: | - cd tests/integration && tox -e lint diff --git a/.github/workflows/security-scan.yaml b/.github/workflows/security-scan.yaml new file mode 100644 index 000000000..044b6dc75 --- /dev/null +++ b/.github/workflows/security-scan.yaml @@ -0,0 +1,81 @@ +name: Security scan + +permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + +on: + workflow_call: + inputs: + # Download k8s-snap using either a GH action artifact or a snap channel. + artifact: + description: The name of a GH action artifact. + type: string + channel: + description: k8s snap channel. + type: string + checkout-ref: + description: k8s-snap git checkout ref, optional. + type: string + +jobs: + security-scan: + name: Security scan + runs-on: ubuntu-latest + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + # We run into rate limiting issues if we don't authenticate + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # Need to fetch the GH actions first. + - name: Checking out repo + uses: actions/checkout@v4 + - name: Download k8s-snap + id: download-snap + uses: ./.github/actions/download-k8s-snap + with: + channel: ${{ inputs.channel }} + artifact: ${{ inputs.artifact }} + - name: Checking out tested repo branch + uses: actions/checkout@v4 + with: + ref: ${{ inputs.checkout-ref }} + # Persist downloaded artifacts + clean: 'false' + # TODO: move the following steps to a separate script + - name: Setup Trivy vulnerability scanner + run: | + mkdir -p manual-trivy/sarifs + pushd manual-trivy + VER=$(curl --silent -qI https://github.com/aquasecurity/trivy/releases/latest | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}'); + wget https://github.com/aquasecurity/trivy/releases/download/${VER}/trivy_${VER#v}_Linux-64bit.tar.gz + tar -zxvf ./trivy_${VER#v}_Linux-64bit.tar.gz + popd + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@master + with: + scan-type: "fs" + ignore-unfixed: true + format: "sarif" + output: "trivy-k8s-repo-scan--results.sarif" + severity: "MEDIUM,HIGH,CRITICAL" + env: + TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db" + - name: Gather Trivy repo scan results + run: | + cp trivy-k8s-repo-scan--results.sarif ./manual-trivy/sarifs/ + - name: Run Trivy vulnerability scanner on the snap + run: | + for var in $(env | grep -o '^TRIVY_[^=]*'); do + unset "$var" + done + cp ${{ steps.download-snap.outputs.snap-path }} ./k8s-test.snap + unsquashfs k8s-test.snap + ./manual-trivy/trivy --db-repository public.ecr.aws/aquasecurity/trivy-db rootfs ./squashfs-root/ --format sarif > ./manual-trivy/sarifs/snap.sarif + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "./manual-trivy/sarifs" diff --git a/.github/workflows/update-branches.yaml b/.github/workflows/update-branches.yaml index 432e4c8f3..3ac56592b 100644 --- a/.github/workflows/update-branches.yaml +++ b/.github/workflows/update-branches.yaml @@ -14,7 +14,7 @@ jobs: name: "${{ matrix.patch }}" permissions: contents: write # for Git to git push - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: patch: ["strict", "moonray"] diff --git a/k8s/scripts/inspect.sh b/k8s/scripts/inspect.sh index 8adcb27f9..33f3344f4 100755 --- a/k8s/scripts/inspect.sh +++ b/k8s/scripts/inspect.sh @@ -54,6 +54,7 @@ function collect_args { function collect_cluster_info { log_info "Copy k8s cluster-info dump to the final report tarball" + # TODO: add a verbose mode that collects logs from all namespaces (--all-namespaces). k8s kubectl cluster-info dump --output-directory "$INSPECT_DUMP/cluster-info" &>/dev/null } diff --git a/tests/integration/tests/test_bootstrap.py b/tests/integration/tests/test_bootstrap.py index 14d2c42b3..0037ae7fb 100644 --- a/tests/integration/tests/test_bootstrap.py +++ b/tests/integration/tests/test_bootstrap.py @@ -15,3 +15,5 @@ def test_microk8s_installed(instances: List[harness.Instance]): instance.exec("snap install microk8s --classic".split()) result = instance.exec("k8s bootstrap".split(), capture_output=True, check=False) assert "Error: microk8s snap is installed" in result.stderr.decode() + + instance.exec("snap remove microk8s --purge".split()) diff --git a/tests/integration/tests/test_cleanup.py b/tests/integration/tests/test_cleanup.py index 9633fb518..be32d4719 100644 --- a/tests/integration/tests/test_cleanup.py +++ b/tests/integration/tests/test_cleanup.py @@ -114,6 +114,7 @@ def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]): @pytest.mark.node_count(1) @pytest.mark.no_setup() @pytest.mark.tags(tags.NIGHTLY) +@pytest.mark.skip(reason="the test fails when using a harness other than 'local'") def test_containerd_path_cleanup_on_failed_init( instances: List[harness.Instance], tmp_path ): diff --git a/tests/integration/tests/test_ingress.py b/tests/integration/tests/test_ingress.py index 1d10a8069..fc6a4e55d 100644 --- a/tests/integration/tests/test_ingress.py +++ b/tests/integration/tests/test_ingress.py @@ -41,7 +41,7 @@ def get_ingress_service_node_port(p): def get_external_service_ip(instance: harness.Instance, service_namespace) -> str: try_count = 0 ingress_ip = None - while ingress_ip is None and try_count < 5: + while ingress_ip is None and try_count < 20: try_count += 1 for svcns in service_namespace: svc = svcns["service"] @@ -91,7 +91,7 @@ def test_ingress(instances: List[harness.Instance]): util.wait_for_dns(instance) result = ( - util.stubbornly(retries=7, delay_s=3) + util.stubbornly(retries=20, delay_s=3) .on(instance) .until(lambda p: get_ingress_service_node_port(p) is not None) .exec(["k8s", "kubectl", "get", "service", "-A", "-o", "json"]) @@ -99,7 +99,7 @@ def test_ingress(instances: List[harness.Instance]): ingress_http_port = get_ingress_service_node_port(result) - assert ingress_http_port is not None, "No ingress nodePort found." + assert ingress_http_port, "No ingress nodePort found." manifest = MANIFESTS_DIR / "ingress-test.yaml" instance.exec( @@ -139,7 +139,7 @@ def test_ingress(instances: List[harness.Instance]): {"service": "cilium-ingress", "namespace": "kube-system"}, ], ) - assert ingress_ip is not None, "No ingress IP found." + assert ingress_ip, "No ingress IP found." util.stubbornly(retries=10, delay_s=5).on(instance).until( lambda p: "Welcome to nginx!" in p.stdout.decode() ).exec(["curl", f"{ingress_ip}", "-H", "Host: foo.bar.com"])