From bfe78922c489b59d8911f1a65dc1c15c96a25704 Mon Sep 17 00:00:00 2001 From: Hongli Lai Date: Sun, 13 Oct 2024 14:26:13 +0000 Subject: [PATCH] Cache generic Linux binaries CI, make architecture name handling consistent --- .github/workflows/binaries.yml | 80 ++++++++++++++++++++++++------- dev/ci/tests/binaries/Jenkinsfile | 13 ++--- dev/ci/tests/binaries/build-linux | 4 +- dev/ci/tests/binaries/test-linux | 4 +- packaging/binaries | 2 +- 5 files changed, 77 insertions(+), 26 deletions(-) diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index d17279577b..4271d803ef 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -18,26 +18,72 @@ jobs: include: - runner: ubuntu-24.04 arch: x86_64 + docker_arch: amd64 - runner: passenger-ubuntu-24.04-arm64-4cpu - arch: arm64 + arch: aarch64 + docker_arch: arm64 runs-on: ${{ matrix.runner }} - env: - WORKSPACE: ${{ github.workspace }} - OUTPUT_DIR: ${{ github.workspace }}/output-linux-${{ matrix.arch }} - ARCHITECTURE: ${{ matrix.arch }} - CACHE_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }} - RUNTIME_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: docker/setup-buildx-action@v3 - - run: ./dev/ci/tests/binaries/build-linux - - uses: actions/upload-artifact@v4 - with: - name: binaries-linux-${{ matrix.arch }} - path: 'output-linux-${{ matrix.arch }}/**/*' - - run: ./dev/ci/tests/binaries/test-linux + - uses: actions/checkout@v4 + with: + submodules: true + + - run: mkdir cache output + working-directory: packaging/binaries/linux + + - name: Fetch cache + run: ./dev/ci/fetch-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/linux-binaries-cache-${{ hashFiles('packaging/binaries/linux/docker_image/Dockerfile') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" + CACHE_PATH: packaging/binaries/linux/cache + + - name: Build + run: > + ./build + -p "$WORKSPACE" + -c cache + -o output + -A "$DOCKER_ARCH" + -j 2 + passenger nginx + env: + WORKSPACE: ${{ github.workspace }} + DOCKER_ARCH: ${{ matrix.docker_arch }} + working-directory: packaging/binaries/linux + + - uses: actions/upload-artifact@v4 + with: + name: binaries-linux-${{ matrix.arch }} + path: packages/binaries/linux/output/**/* + + - name: Update cache + run: ./dev/ci/update-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/linux-binaries-cache-${{ hashFiles('packaging/binaries/linux/docker_image/Dockerfile') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" + CACHE_PATH: packaging/binaries/linux/cache + if: always() + + - name: Package + run: ./package -i output -o output -a "$ARCHITECTURE" + env: + ARCHITECTURE: ${{ matrix.arch }} + working-directory: packaging/binaries/linux + + - name: Test + run: > + ./test + -p "$WORKSPACE" + -i output + -I output + -A "$DOCKER_ARCH" + env: + WORKSPACE: ${{ github.workspace }} + DOCKER_ARCH: ${{ matrix.docker_arch }} + working-directory: packaging/binaries/linux build_macos: name: Binaries macOS ${{ matrix.arch }} diff --git a/dev/ci/tests/binaries/Jenkinsfile b/dev/ci/tests/binaries/Jenkinsfile index a735b34d66..1fa437ce4f 100644 --- a/dev/ci/tests/binaries/Jenkinsfile +++ b/dev/ci/tests/binaries/Jenkinsfile @@ -1,12 +1,13 @@ JOB_NAME_AS_ID = null -def setupLinuxTest(enablerFlag, architecture, block) { +def setupLinuxTest(enablerFlag, architecture, dockerArch, block) { if (enablerFlag) { node("linux && ${architecture}") { withEnv([ "OUTPUT_DIR=${env.WORKSPACE}/output-linux-${architecture}", "CACHE_DIR=${env.JENKINS_HOME}/cache/${env.JOB_NAME_AS_ID}/linux-${architecture}/executor-${env.EXECUTOR_NUMBER}", - "ARCHITECTURE=${architecture}" + "ARCHITECTURE=${architecture}", + "DOCKER_ARCH=${dockerArch}" ], block) } } else { @@ -41,7 +42,7 @@ pipeline { parameters { booleanParam(name: 'LINUX_X86_64', defaultValue: true, description: 'Linux x86_64 binaries') - booleanParam(name: 'LINUX_ARM64', defaultValue: true, description: 'Linux arm64 binaries') + booleanParam(name: 'LINUX_AARCH64', defaultValue: true, description: 'Linux aarch64 binaries') booleanParam(name: 'MACOS', defaultValue: true, description: 'macOS binaries') } @@ -73,15 +74,15 @@ pipeline { script { parallel( 'Linux x86_64': { - setupLinuxTest(params.LINUX_X86_64, 'x86_64') { + setupLinuxTest(params.LINUX_X86_64, 'x86_64', 'amd64') { checkout scm sh './dev/ci/tests/binaries/build-linux' archiveArtifacts artifacts: 'output-linux-x86_64/**/*' sh './dev/ci/tests/binaries/test-linux' } }, - 'Linux arm64': { - setupLinuxTest(params.LINUX_ARM64, 'aarch64') { + 'Linux aarch64': { + setupLinuxTest(params.LINUX_AARCH64, 'aarch64', 'arm64') { checkout scm sh './dev/ci/tests/binaries/build-linux' archiveArtifacts artifacts: 'output-linux-aarch64/**/*' diff --git a/dev/ci/tests/binaries/build-linux b/dev/ci/tests/binaries/build-linux index e9c787ed47..1d1f9d373f 100755 --- a/dev/ci/tests/binaries/build-linux +++ b/dev/ci/tests/binaries/build-linux @@ -6,6 +6,7 @@ # # WORKSPACE # ARCHITECTURE +# DOCKER_ARCH # # Optional environment variables: # @@ -21,6 +22,7 @@ source "./shared/lib/library.sh" require_envvar WORKSPACE "$WORKSPACE" require_envvar ARCHITECTURE "$ARCHITECTURE" +require_envvar DOCKER_ARCH "$DOCKER_ARCH" PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}" OUTPUT_DIR="${OUTPUT_DIR:-$WORKSPACE/output}" @@ -33,6 +35,6 @@ run ./linux/build \ -p "$PASSENGER_ROOT" \ -c "$CACHE_DIR" \ -o "$OUTPUT_DIR" \ - -a "$ARCHITECTURE" \ + -A "$DOCKER_ARCH" \ -j 1 \ passenger nginx diff --git a/dev/ci/tests/binaries/test-linux b/dev/ci/tests/binaries/test-linux index 9bea870a66..6a63f68c48 100755 --- a/dev/ci/tests/binaries/test-linux +++ b/dev/ci/tests/binaries/test-linux @@ -6,6 +6,7 @@ # # WORKSPACE # ARCHITECTURE +# DOCKER_ARCH # # Optional environment variables: # @@ -21,6 +22,7 @@ source "./shared/lib/library.sh" require_envvar WORKSPACE "$WORKSPACE" require_envvar ARCHITECTURE "$ARCHITECTURE" +require_envvar DOCKER_ARCH "$DOCKER_ARCH" PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}" OUTPUT_DIR="${OUTPUT_DIR:-$WORKSPACE/output}" @@ -41,5 +43,5 @@ run ./linux/test \ -p "$PASSENGER_ROOT" \ -i "$OUTPUT_DIR" \ -I "$OUTPUT_DIR" \ - -a "$ARCHITECTURE" \ + -A "$DOCKER_ARCH" \ "${EXTRA_TEST_PARAMS[@]}" diff --git a/packaging/binaries b/packaging/binaries index 18054db278..9b4cc8f6cf 160000 --- a/packaging/binaries +++ b/packaging/binaries @@ -1 +1 @@ -Subproject commit 18054db2780b939be2ffd1ea7004b4ef14fb0a07 +Subproject commit 9b4cc8f6cf7ffcbfe01c72fdc3ffe28d3007860a