build #266
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| schedule: | |
| - cron: '0 10 * * *' # everyday at 10am | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - 'master' | |
| - 'v[0-9]+.[0-9]+' | |
| tags: | |
| # Only accept tags in the format: vX.Y.Z_<serial> | |
| - 'v*.*.*_*' | |
| # - 'dockerfile/*' | |
| pull_request: | |
| paths-ignore: | |
| - 'README.md' | |
| - 'docs/**' | |
| - 'frontend/dockerfile/docs/**' | |
| env: | |
| # it is ok to the the upstream image here, beacause the REPO_SLUG_ORIGIN is | |
| # only used to set the buildx context for the steps where other builds and | |
| # tests will take place, like the Ubuntu-based builds | |
| REPO_SLUG_ORIGIN: "moby/buildkit:v0.11.0-rc4" | |
| # this is the one that matters, as it is our desired rebased output | |
| REPO_SLUG_TARGET: ${{ startsWith(github.ref, 'refs/tags/v') && secrets.ARTIFACTORY_REGISTRY_REPO || 'ghcr.io/canonical/buildkit' }} | |
| # we aren't gonna touch this | |
| DF_REPO_SLUG_TARGET: "docker/dockerfile-upstream" | |
| # PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64" | |
| PLATFORMS: "linux/amd64" | |
| CACHE_GHA_SCOPE_IT: "integration-tests" | |
| CACHE_GHA_SCOPE_BINARIES: "binaries" | |
| CACHE_GHA_SCOPE_CROSS: "cross" | |
| TESTFLAGS: "-v --parallel=6 --timeout=30m" | |
| BUILDX_VERSION: "v0.10.0-rc3" # leave empty to use the one available on GitHub virtual environment | |
| GO_VERSION: "1.19" | |
| jobs: | |
| base: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - | |
| name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| version: ${{ env.BUILDX_VERSION }} | |
| driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| buildkitd-flags: --debug | |
| - | |
| name: Build ${{ env.CACHE_GHA_SCOPE_BINARIES }} | |
| run: | | |
| ./hack/build_ci_first_pass binaries | |
| env: | |
| CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} | |
| CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} | |
| - | |
| name: Build ${{ env.CACHE_GHA_SCOPE_IT }} | |
| run: | | |
| ./hack/build_ci_first_pass integration-tests | |
| env: | |
| CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} | |
| CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} | |
| test: | |
| runs-on: ubuntu-20.04 | |
| needs: [base] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| pkg: | |
| - ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend | |
| # - ./frontend/dockerfile | |
| worker: | |
| - containerd | |
| # - containerd-rootless | |
| - containerd-1.5 | |
| - containerd-snapshotter-stargz | |
| - oci | |
| # - oci-rootless | |
| - oci-snapshotter-stargz | |
| typ: | |
| - integration | |
| - dockerfile | |
| exclude: | |
| - pkg: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend | |
| typ: dockerfile | |
| include: | |
| - pkg: ./... | |
| skip-integration-tests: 1 | |
| typ: integration gateway | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - | |
| name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| version: ${{ env.BUILDX_VERSION }} | |
| driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| buildkitd-flags: --debug | |
| - | |
| name: Test pkg=${{ matrix.pkg }} ; typ=${{ matrix.typ }} ; skipit=${{ matrix.skip-integration-tests }} ; worker=${{ matrix.worker }} | |
| run: | | |
| if [ -n "${{ matrix.worker }}" ]; then | |
| export TESTFLAGS="${TESTFLAGS} --run=//worker=${{ matrix.worker }}$" | |
| fi | |
| ./hack/test ${{ matrix.typ }} | |
| mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.typ }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt | |
| env: | |
| TEST_COVERAGE: 1 | |
| TESTPKGS: ${{ matrix.pkg }} | |
| SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} | |
| CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} | |
| - | |
| name: Upload coverage file | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: ./coverage | |
| test-nydus: | |
| runs-on: ubuntu-20.04 | |
| needs: [base] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| pkg: | |
| - ./client | |
| worker: | |
| - containerd | |
| - oci | |
| typ: | |
| - integration | |
| exclude: | |
| - pkg: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend | |
| typ: dockerfile | |
| include: | |
| - pkg: ./... | |
| skip-integration-tests: 1 | |
| typ: integration | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - | |
| name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| version: ${{ env.BUILDX_VERSION }} | |
| driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| buildkitd-flags: --debug | |
| - | |
| name: Test pkg=${{ matrix.pkg }} ; typ=${{ matrix.typ }} ; skipit=${{ matrix.skip-integration-tests }} ; worker=${{ matrix.worker }} | |
| run: | | |
| if [ -n "${{ matrix.worker }}" ]; then | |
| export TESTFLAGS="${TESTFLAGS} --tags=nydus --run=//worker=${{ matrix.worker }}$" | |
| fi | |
| ./hack/test ${{ matrix.typ }} | |
| env: | |
| BUILDKITD_TAGS: nydus | |
| TESTPKGS: ${{ matrix.pkg }} | |
| SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} | |
| CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} | |
| test-s3: | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - base | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| version: ${{ env.BUILDX_VERSION }} | |
| driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| buildkitd-flags: --debug | |
| - | |
| name: Test | |
| run: | | |
| hack/s3_test/run_test.sh | |
| env: | |
| ARTIFACTORY_APT_AUTH_CONF: ${{ secrets.ARTIFACTORY_APT_AUTH_CONF }} | |
| ARTIFACTORY_BASE64_GPG: ${{ secrets.ARTIFACTORY_BASE64_GPG }} | |
| test-os: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| # - ubuntu-20.04 | |
| # - macOS-11 | |
| - windows-2022 | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Set up Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - | |
| name: Go mod | |
| run: | | |
| go mod download | |
| - | |
| name: Test | |
| env: | |
| TMPDIR: ${{ runner.temp }} | |
| SKIP_INTEGRATION_TESTS: 1 | |
| run: | | |
| mkdir -p ./coverage | |
| go test -coverprofile=./coverage/coverage-${{ github.job }}-${{ matrix.os }}.txt -covermode=atomic ${TESTFLAGS} ./... | |
| shell: bash | |
| - | |
| name: Upload coverage file | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: ./coverage | |
| upload-coverage: | |
| runs-on: ubuntu-20.04 | |
| needs: [test, test-os] | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Download coverage files | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage | |
| path: ./coverage | |
| - | |
| name: List coverage files | |
| uses: actions/github-script@v6 | |
| id: files | |
| with: | |
| result-encoding: string | |
| script: | | |
| return require('fs').readdirSync('./coverage', {withFileTypes: true}) | |
| .filter(item => !item.isDirectory()) | |
| .map(item => `./coverage/${item.name}`) | |
| .join(','); | |
| - | |
| name: Send to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| files: ${{ steps.files.outputs.result }} | |
| cross: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - | |
| name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| version: ${{ env.BUILDX_VERSION }} | |
| driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| buildkitd-flags: --debug | |
| - | |
| name: Cross | |
| run: | | |
| ./hack/cross | |
| env: | |
| # PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 | |
| # we're only building for Linux | |
| PLATFORMS: ${{ env.PLATFORMS }} | |
| RUNC_PLATFORMS: ${{ env.PLATFORMS }} | |
| CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} | |
| CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} | |
| ARTIFACTORY_APT_AUTH_CONF: ${{ secrets.ARTIFACTORY_APT_AUTH_CONF }} | |
| ARTIFACTORY_BASE64_GPG: ${{ secrets.ARTIFACTORY_BASE64_GPG }} | |
| release-base: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| tag: ${{ steps.prep.outputs.tag }} | |
| push: ${{ steps.prep.outputs.push }} | |
| steps: | |
| - name: Prepare | |
| id: prep | |
| run: | | |
| TAG=pr | |
| PUSH=false | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| TAG=nightly | |
| PUSH=push | |
| elif [[ $GITHUB_REF == refs/tags/v* ]]; then | |
| TAG=${GITHUB_REF#refs/tags/} | |
| PUSH=push | |
| elif [[ $GITHUB_REF == refs/heads/* ]]; then | |
| TAG=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') | |
| if [ $GITHUB_REF = "refs/heads/${{ github.event.repository.default_branch }}" ]; then | |
| PUSH=push | |
| fi | |
| fi | |
| echo "tag=${TAG}" >>${GITHUB_OUTPUT} | |
| echo "push=${PUSH}" >>${GITHUB_OUTPUT} | |
| image: | |
| runs-on: ubuntu-20.04 | |
| needs: [release-base, test, cross] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target-stage: | |
| - '' | |
| # - rootless | |
| env: | |
| TARGET: ${{ matrix.target-stage }} | |
| RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| CACHE_TO: type=gha,scope=image${{ matrix.target-stage }} | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - | |
| name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| id: setup-buildx-builder | |
| with: | |
| version: ${{ env.BUILDX_VERSION }} | |
| driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| buildkitd-flags: --debug | |
| - | |
| name: Login to GHCR | |
| if: needs.release-base.outputs.push == 'push' | |
| uses: docker/login-action@v2 | |
| env: | |
| REGISTRY: ${{ startsWith(github.ref, 'refs/tags/v') && secrets.ARTIFACTORY_REGISTRY || 'ghcr.io' }} | |
| USERNAME: ${{ startsWith(github.ref, 'refs/tags/v') && secrets.ARTIFACTORY_USER || github.actor }} | |
| PASSWORD: ${{ startsWith(github.ref, 'refs/tags/v') && secrets.ARTIFACTORY_ACCESS_TOKEN || secrets.GITHUB_TOKEN }} | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ env.USERNAME }} | |
| password: ${{ env.PASSWORD }} | |
| - | |
| name: Build local image for testing | |
| run: | | |
| ./hack/images local "$REPO_SLUG_TARGET" "nopush" | |
| env: | |
| # have CACHE_FROM here cause the "env" context is not available at the job level | |
| CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} type=gha,scope=image${{ matrix.target-stage }}" | |
| ARTIFACTORY_ACCESS_TOKEN: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} | |
| ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }} | |
| ARTIFACTORY_APT_AUTH_CONF: ${{ secrets.ARTIFACTORY_APT_AUTH_CONF }} | |
| ARTIFACTORY_BASE64_GPG: ${{ secrets.ARTIFACTORY_BASE64_GPG }} | |
| - | |
| name: Test buildkit image locally before pushing | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install skopeo | |
| ./hack/canonical_test/run_test.sh | |
| env: | |
| IMG_NAME: '${{ env.REPO_SLUG_TARGET }}:local' | |
| - | |
| name: Push ${{ needs.release-base.outputs.tag }} to GHCR | |
| if: needs.release-base.outputs.push == 'push' | |
| run: | | |
| docker buildx use ${{ steps.setup-buildx-builder.outputs.name }} | |
| ./hack/images "${{ needs.release-base.outputs.tag }}" "$REPO_SLUG_TARGET" push | |
| env: | |
| # have CACHE_FROM here cause the "env" context is not available at the job level | |
| CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} type=gha,scope=image${{ matrix.target-stage }}" | |
| ARTIFACTORY_ACCESS_TOKEN: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} | |
| ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }} | |
| ARTIFACTORY_APT_AUTH_CONF: ${{ secrets.ARTIFACTORY_APT_AUTH_CONF }} | |
| ARTIFACTORY_BASE64_GPG: ${{ secrets.ARTIFACTORY_BASE64_GPG }} | |
| binaries: | |
| runs-on: ubuntu-20.04 | |
| needs: [release-base, test, cross] | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v3 | |
| - | |
| name: Expose GitHub Runtime | |
| uses: crazy-max/ghaction-github-runtime@v2 | |
| - | |
| name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| version: ${{ env.BUILDX_VERSION }} | |
| driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| buildkitd-flags: --debug | |
| - | |
| name: Build ${{ needs.release-base.outputs.tag }} | |
| run: | | |
| ./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out | |
| env: | |
| RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| # PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 | |
| # we're only building for Linux | |
| PLATFORMS: ${{ env.PLATFORMS }} | |
| CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} | |
| - | |
| name: Upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: buildkit | |
| path: ./release-out/* | |
| if-no-files-found: error | |
| - | |
| name: GitHub Release | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| draft: true | |
| files: ./release-out/* | |
| name: ${{ needs.release-base.outputs.tag }} | |
| # we don't rebase/release the frontend...just the buildkit | |
| # frontend-base: | |
| # runs-on: ubuntu-20.04 | |
| # if: github.event_name != 'schedule' | |
| # outputs: | |
| # typ: ${{ steps.prep.outputs.typ }} | |
| # push: ${{ steps.prep.outputs.push }} | |
| # matrix: ${{ steps.prep.outputs.matrix }} | |
| # steps: | |
| # - | |
| # name: Prepare | |
| # id: prep | |
| # run: | | |
| # TYP=master | |
| # TAG=mainline | |
| # PUSH=false | |
| # if [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then | |
| # TYP=tag | |
| # TAG=${GITHUB_REF#refs/tags/} | |
| # PUSH=push | |
| # elif [ $GITHUB_REF = "refs/heads/${{ github.event.repository.default_branch }}" ]; then | |
| # PUSH=push | |
| # fi | |
| # echo "typ=${TYP}" >>${GITHUB_OUTPUT} | |
| # echo "push=${PUSH}" >>${GITHUB_OUTPUT} | |
| # if [ "${TYP}" = "master" ]; then | |
| # echo "matrix=$(jq -cn --arg tag "$TAG" '[$tag, "labs"]')" >>${GITHUB_OUTPUT} | |
| # else | |
| # echo "matrix=$(jq -cn --arg tag "$TAG" '[$tag]')" >>${GITHUB_OUTPUT} | |
| # fi | |
| # frontend-image: | |
| # runs-on: ubuntu-20.04 | |
| # if: github.event_name != 'schedule' | |
| # needs: [frontend-base, test] | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # tag: ${{ fromJson(needs.frontend-base.outputs.matrix) }} | |
| # steps: | |
| # - | |
| # name: Prepare | |
| # run: | | |
| # if [ "${{ matrix.tag }}" = "labs" ]; then | |
| # echo "CACHE_SCOPE=frontend-labs" >>${GITHUB_ENV} | |
| # else | |
| # echo "CACHE_SCOPE=frontend-mainline" >>${GITHUB_ENV} | |
| # fi | |
| # - | |
| # name: Checkout | |
| # uses: actions/checkout@v3 | |
| # - | |
| # name: Expose GitHub Runtime | |
| # uses: crazy-max/ghaction-github-runtime@v2 | |
| # - | |
| # name: Set up QEMU | |
| # uses: docker/setup-qemu-action@v2 | |
| # - | |
| # name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v2 | |
| # with: | |
| # version: ${{ env.BUILDX_VERSION }} | |
| # driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} | |
| # buildkitd-flags: --debug | |
| # - | |
| # name: Login to DockerHub | |
| # uses: docker/login-action@v2 | |
| # if: needs.frontend-base.outputs.push == 'push' | |
| # with: | |
| # username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| # password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # - | |
| # name: Build | |
| # run: | | |
| # ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ matrix.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" | |
| # env: | |
| # RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| # PLATFORMS: ${{ env.PLATFORMS }},linux/386,linux/mips,linux/mipsle,linux/mips64,linux/mips64le | |
| # CACHE_FROM: type=gha,scope=${{ env.CACHE_SCOPE }} | |
| # CACHE_TO: type=gha,scope=${{ env.CACHE_SCOPE }} |