From cee2aea51f9340bbecfdac025ed959e09e3f6d7d Mon Sep 17 00:00:00 2001 From: Jeroen Op 't Eynde Date: Fri, 13 Dec 2024 15:03:18 +0100 Subject: [PATCH] chore: update build/, Makefile and integration test script (#204) Signed-off-by: Duologic --- .github/workflows/backport.yml | 28 +++++ .github/workflows/ci.yml | 159 +++++++++++++++-------------- .github/workflows/codeql.yml | 42 ++++++++ .github/workflows/commands.yml | 86 ++++++++++++++++ .github/workflows/promote.yml | 9 +- .github/workflows/tag.yml | 4 +- .gitmodules | 2 +- Makefile | 14 +-- build | 2 +- cluster/local/integration_tests.sh | 35 ++++--- 10 files changed, 272 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/backport.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/commands.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 00000000..1ee4208e --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,28 @@ +name: Backport + +on: + # NOTE(negz): This is a risky target, but we run this action only when and if + # a PR is closed, then filter down to specifically merged PRs. We also don't + # invoke any scripts, etc from within the repo. I believe the fact that we'll + # be able to review PRs before this runs makes this fairly safe. + # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + pull_request_target: + types: [closed] + # See also commands.yml for the /backport triggered variant of this workflow. + +jobs: + # NOTE(negz): I tested many backport GitHub actions before landing on this + # one. Many do not support merge commits, or do not support pull requests with + # more than one commit. This one does. It also handily links backport PRs with + # new PRs, and provides commentary and instructions when it can't backport. + # The main gotcha with this action is that PRs _must_ be labelled before they're + # merged to trigger a backport. + open-pr: + runs-on: ubuntu-22.04 + if: github.event.pull_request.merged + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Open Backport PR + uses: korthout/backport-action@v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96a4218c..f78a81d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ env: # Common versions GO_VERSION: '1.21' GOLANGCI_VERSION: 'v1.54.0' - DOCKER_BUILDX_VERSION: 'v0.8.2' + DOCKER_BUILDX_VERSION: 'v0.9.1' # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether @@ -23,13 +23,13 @@ env: jobs: detect-noop: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: noop: ${{ steps.noop.outputs.should_skip }} steps: - name: Detect No-op Changes id: noop - uses: fkirc/skip-duplicate-actions@v2.1.0 + uses: fkirc/skip-duplicate-actions@v5.2.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} paths_ignore: '["**.md", "**.png", "**.jpg"]' @@ -38,45 +38,44 @@ jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cachedir=$(make go.cachedir)" >> $GITHUB_ENV - name: Cache the Go Build Cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: - path: ${{ steps.go.outputs.cache }} + path: ${{ env.cachedir }} key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-lint- - name: Cache Go Dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-pkg- - - name: Vendor Dependencies - run: make vendor vendor.check + - name: Download Go Modules + run: make modules.download modules.check - # We could run 'make lint' to ensure our desired Go version, but we prefer - # this action because it leaves 'annotations' (i.e. it comments on PRs to - # point out linter violations). + # We could run 'make lint' but we prefer this action because it leaves + # 'annotations' (i.e. it comments on PRs to point out linter violations). - name: Lint uses: golangci/golangci-lint-action@v3 with: @@ -84,53 +83,60 @@ jobs: skip-go-installation: true check-diff: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cachedir=$(make go.cachedir)" >> $GITHUB_ENV - name: Cache the Go Build Cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: - path: ${{ steps.go.outputs.cache }} + path: ${{ env.cachedir }} key: ${{ runner.os }}-build-check-diff-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-check-diff- - name: Cache Go Dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-pkg- - - name: Vendor Dependencies - run: make vendor vendor.check + - name: Download Go Modules + run: make modules.download modules.check - name: Check Diff - run: make check-diff + id: check-diff + run: | + mkdir _output + make check-diff + + - name: Show diff + if: failure() && steps.check-diff.outcome == 'failure' + run: git diff unit-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true @@ -138,59 +144,59 @@ jobs: run: git fetch --prune --unshallow - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cachedir=$(make go.cachedir)" >> $GITHUB_ENV - name: Cache the Go Build Cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: - path: ${{ steps.go.outputs.cache }} + path: ${{ env.cachedir }} key: ${{ runner.os }}-build-unit-tests-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-unit-tests- - name: Cache Go Dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-pkg- - - name: Vendor Dependencies - run: make vendor vendor.check + - name: Download Go Modules + run: make modules.download modules.check - name: Run Unit Tests run: make -j2 test - name: Publish Unit Test Coverage - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: flags: unittests file: _output/tests/linux_amd64/coverage.txt e2e-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' steps: - name: Setup QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 with: platforms: all - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: ${{ env.DOCKER_BUILDX_VERSION }} install: true - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true @@ -198,61 +204,68 @@ jobs: run: git fetch --prune --unshallow - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cachedir=$(make go.cachedir)" >> $GITHUB_ENV - name: Cache the Go Build Cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-e2e-tests-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-e2e-tests- + path: ${{ env.cachedir }} + key: ${{ runner.os }}-build-unit-tests-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-build-unit-tests- - name: Cache Go Dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-pkg- + restore-keys: ${{ runner.os }}-pkg- - - name: Vendor Dependencies - run: make vendor vendor.check + - name: Download Go Modules + run: make modules.download modules.check - name: Build Helm Chart run: make -j2 build - env: - # We're using docker buildx, which doesn't actually load the images it - # builds by default. Specifying --load does so. - BUILD_ARGS: "--load" + #env: + # # We're using docker buildx, which doesn't actually load the images it + # # builds by default. Specifying --load does so. + # BUILD_ARGS: "--load" - name: Run E2E Tests - run: make e2e USE_HELM3=true + run: make e2e USE_HELM=true publish-artifacts: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' steps: - name: Setup QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 with: platforms: all - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: ${{ env.DOCKER_BUILDX_VERSION }} install: true + - name: Login to Upbound + uses: docker/login-action@v1 + if: env.XPKG_ACCESS_ID != '' + with: + registry: xpkg.upbound.io + username: ${{ secrets.XPKG_ACCESS_ID }} + password: ${{ secrets.XPKG_TOKEN }} + - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true @@ -260,30 +273,30 @@ jobs: run: git fetch --prune --unshallow - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} - name: Find the Go Build Cache id: go - run: echo "::set-output name=cache::$(make go.cachedir)" + run: echo "cachedir=$(make go.cachedir)" >> $GITHUB_ENV - name: Cache the Go Build Cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: - path: ${{ steps.go.outputs.cache }} + path: ${{ env.cachedir }} key: ${{ runner.os }}-build-publish-artifacts-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-build-publish-artifacts- - name: Cache Go Dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: .work/pkg key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-pkg- - - name: Vendor Dependencies - run: make vendor vendor.check + - name: Download Go Modules + run: make modules.download modules.check - name: Build Artifacts run: make -j2 build.all @@ -293,11 +306,15 @@ jobs: BUILD_ARGS: "--load" - name: Publish Artifacts to GitHub - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: output path: _output/** + - name: Publish Artifacts + if: env.XPKG_ACCESS_ID != '' + run: make publish BRANCH_NAME=${GITHUB_REF##*/} + - name: Login to Docker uses: docker/login-action@v1 if: env.CONTRIB_DOCKER_USR != '' @@ -305,14 +322,6 @@ jobs: username: ${{ secrets.CONTRIB_DOCKER_USR }} password: ${{ secrets.CONTRIB_DOCKER_PSW }} - - name: Login to Upbound - uses: docker/login-action@v1 - if: env.XPKG_ACCESS_ID != '' - with: - registry: xpkg.upbound.io - username: ${{ secrets.XPKG_ACCESS_ID }} - password: ${{ secrets.XPKG_TOKEN }} - - name: Publish Artifacts to S3 and Docker Hub run: make -j2 publish BRANCH_NAME=${GITHUB_REF##*/} if: env.AWS_USR != '' && env.CONTRIB_DOCKER_USR != '' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..6b077c85 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,42 @@ +name: CodeQL + +on: + push: + branches: + - master + - release-* + workflow_dispatch: {} + +jobs: + detect-noop: + runs-on: ubuntu-22.04 + outputs: + noop: ${{ steps.noop.outputs.should_skip }} + steps: + - name: Detect No-op Changes + id: noop + uses: fkirc/skip-duplicate-actions@v5.2.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + paths_ignore: '["**.md", "**.png", "**.jpg"]' + do_not_skip: '["workflow_dispatch", "schedule", "push"]' + concurrent_skipping: false + + analyze: + runs-on: ubuntu-22.04 + needs: detect-noop + if: needs.detect-noop.outputs.noop != 'true' + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: go + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml new file mode 100644 index 00000000..df101a8e --- /dev/null +++ b/.github/workflows/commands.yml @@ -0,0 +1,86 @@ +name: Comment Commands + +on: issue_comment + +jobs: + points: + runs-on: ubuntu-22.04 + if: startsWith(github.event.comment.body, '/points') + + steps: + - name: Extract Command + id: command + uses: xt0rted/slash-command-action@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + command: points + reaction: "true" + reaction-type: "eyes" + allow-edits: "false" + permission-level: write + - name: Handle Command + uses: actions/github-script@v4 + env: + POINTS: ${{ steps.command.outputs.command-arguments }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const points = process.env.POINTS + + if (isNaN(parseInt(points))) { + console.log("Malformed command - expected '/points '") + github.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: "confused" + }) + return + } + const label = "points/" + points + + // Delete our needs-points-label label. + try { + await github.issues.deleteLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: ['needs-points-label'] + }) + console.log("Deleted 'needs-points-label' label.") + } + catch(e) { + console.log("Label 'needs-points-label' probably didn't exist.") + } + + // Add our points label. + github.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: [label] + }) + console.log("Added '" + label + "' label.") + + # NOTE(negz): See also backport.yml, which is the variant that triggers on PR + # merge rather than on comment. + backport: + runs-on: ubuntu-22.04 + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/backport') + steps: + - name: Extract Command + id: command + uses: xt0rted/slash-command-action@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + command: backport + reaction: "true" + reaction-type: "eyes" + allow-edits: "false" + permission-level: write + + - name: Checkout + uses: actions/checkout@v3 + + - name: Open Backport PR + uses: korthout/backport-action@v1 diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index e297e8ed..680c0f9e 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -9,12 +9,9 @@ on: channel: description: 'Release channel' required: true - default: 'alpha' + default: 'stable' env: - # Common versions - GO_VERSION: '1.18' - # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether # credentials have been provided before trying to run steps that need them. @@ -24,11 +21,11 @@ env: jobs: promote-artifacts: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index 3b272eaf..db32dd0f 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -12,11 +12,11 @@ on: jobs: create-tag: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Create Tag uses: negz/create-tag@v1 diff --git a/.gitmodules b/.gitmodules index c2fad470..8f84209c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "build"] path = build - url = https://github.com/upbound/build + url = https://github.com/crossplane/build diff --git a/Makefile b/Makefile index 6b532fb2..ff3a0507 100644 --- a/Makefile +++ b/Makefile @@ -29,15 +29,10 @@ GO_SUBDIRS += cmd pkg apis GO111MODULE = on -include build/makelib/golang.mk -# kind-related versions -KIND_VERSION ?= v0.12.0 -KIND_NODE_IMAGE_TAG ?= v1.23.4 - # ==================================================================================== # Setup Kubernetes tools - -UP_VERSION = v0.31.0 -UP_CHANNEL = stable +KIND_NODE_IMAGE_TAG ?= v1.23.4 +DOCKER_REGISTRY ?= "xpkg.upbound.io" -include build/makelib/k8s_tools.mk # ==================================================================================== @@ -46,7 +41,6 @@ UP_CHANNEL = stable IMAGES = provider-sql -include build/makelib/imagelight.mk - # ==================================================================================== # Setup XPKG @@ -87,7 +81,7 @@ generate: crds.clean e2e.run: test-integration # Run integration tests. -test-integration: $(KIND) $(KUBECTL) $(UP) $(HELM3) +test-integration: $(KIND) $(KUBECTL) $(UP) $(HELM) @$(INFO) running integration tests using kind $(KIND_VERSION) @KIND_NODE_IMAGE_TAG=${KIND_NODE_IMAGE_TAG} $(ROOT_DIR)/cluster/local/integration_tests.sh || $(FAIL) @$(OK) integration tests passed @@ -124,7 +118,7 @@ dev: $(KIND) $(KUBECTL) @$(KIND) create cluster --name=$(PROJECT_NAME)-dev @$(KUBECTL) cluster-info --context kind-$(PROJECT_NAME)-dev @$(INFO) Installing Crossplane CRDs - @$(KUBECTL) apply -k https://github.com/crossplane/crossplane//cluster?ref=master + @$(KUBECTL) apply --server-side -k https://github.com/crossplane/crossplane//cluster?ref=master @$(INFO) Installing Provider SQL CRDs @$(KUBECTL) apply -R -f package/crds @$(INFO) Starting Provider SQL controllers diff --git a/build b/build index 3b994632..231258db 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit 3b99463225581259ce39c7d7a45290be12515abb +Subproject commit 231258db281237379d8ec0c6e4af9d7c1ae5cc4a diff --git a/cluster/local/integration_tests.sh b/cluster/local/integration_tests.sh index 17ed43e1..fc19b1d5 100755 --- a/cluster/local/integration_tests.sh +++ b/cluster/local/integration_tests.sh @@ -146,33 +146,40 @@ cleanup_cluster() { setup_crossplane() { echo_step "installing crossplane from stable channel" - "${HELM3}" repo add crossplane-stable https://charts.crossplane.io/stable/ --force-update - local chart_version="$("${HELM3}" search repo crossplane-stable/crossplane | awk 'FNR == 2 {print $2}')" + "${HELM}" repo add crossplane-stable https://charts.crossplane.io/stable/ --force-update + local chart_version="$("${HELM}" search repo crossplane-stable/crossplane | awk 'FNR == 2 {print $2}')" echo_info "using crossplane version ${chart_version}" echo # we replace empty dir with our PVC so that the /cache dir in the kind node # container is exposed to the crossplane pod - "${HELM3}" install crossplane --namespace crossplane-system crossplane-stable/crossplane --version ${chart_version} --wait --set packageCache.pvc=package-cache + "${HELM}" install crossplane --namespace crossplane-system crossplane-stable/crossplane --version ${chart_version} --wait --set packageCache.pvc=package-cache } setup_provider() { echo_step "installing provider" local yaml="$( cat </dev/null - "${HELM3}" install mariadb bitnami/mariadb \ + "${HELM}" repo add bitnami https://charts.bitnami.com/bitnami >/dev/null + "${HELM}" install mariadb bitnami/mariadb \ --version 11.3.0 \ --set auth.rootPassword="${MARIADB_ROOT_PW}" \ --wait @@ -355,8 +362,8 @@ initdbScripts: EOF ) - "${HELM3}" repo add bitnami https://charts.bitnami.com/bitnami >/dev/null - "${HELM3}" install mariadb bitnami/mariadb \ + "${HELM}" repo add bitnami https://charts.bitnami.com/bitnami >/dev/null + "${HELM}" install mariadb bitnami/mariadb \ --version 11.3.0 \ --values <(echo "$values") \ --wait @@ -364,7 +371,7 @@ EOF cleanup_mariadb() { echo_step "uninstalling MariaDB" - "${HELM3}" uninstall mariadb + "${HELM}" uninstall mariadb "${KUBECTL}" delete secret mariadb-creds }