Snapshot build #404
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This action enables building container images and executables for farmer and node, can be triggered manually or by | |
| # release creation. | |
| # | |
| # Container images are only pushed to GitHub Container Registry for releases. | |
| # Executables are built both for releases and for manually triggered runs, uploaded to artifacts and assets. | |
| name: Snapshot build | |
| on: | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "snapshot-*" | |
| - "gemini-*" | |
| # Incremental compilation here isn't helpful | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| jobs: | |
| container-linux: | |
| # TODO: Restore once we have self-hosted runners with Ubuntu 22.04 | |
| #runs-on: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-22.04"') }} | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: write | |
| packages: write | |
| strategy: | |
| matrix: | |
| image: | |
| - farmer | |
| - node | |
| - bootstrap-node | |
| fail-fast: false | |
| steps: | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 | |
| - name: Log into registry | |
| uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - name: Extract Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| with: | |
| images: | | |
| ghcr.io/${{ github.repository_owner }}/${{ matrix.image }} | |
| tags: | | |
| type=ref,event=tag | |
| type=ref,event=branch | |
| type=sha,format=long | |
| flavor: | | |
| latest=false | |
| suffix= | |
| - name: Build and push ${{ matrix.image }} image | |
| uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 | |
| with: | |
| file: Dockerfile-${{ matrix.image }} | |
| platforms: linux/amd64,linux/arm64,linux/riscv64 | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| SUBSTRATE_CLI_GIT_COMMIT_HASH=${{ github.sha }} | |
| - name: Trigger trivy-security-scan Workflow | |
| uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # @v3.0.0 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| event-type: trivy-scan-dispatch | |
| client-payload: '{"image": "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:sha-${{ github.sha }}"}' | |
| executables: | |
| strategy: | |
| matrix: | |
| build: | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} | |
| target: x86_64-unknown-linux-gnu | |
| suffix: ubuntu-x86_64-skylake-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=skylake" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} | |
| target: x86_64-unknown-linux-gnu | |
| suffix: ubuntu-x86_64-v2-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=x86-64-v2" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} | |
| target: aarch64-unknown-linux-gnu | |
| suffix: ubuntu-aarch64-${{ github.ref_name }} | |
| rustflags: "-C linker=aarch64-linux-gnu-gcc" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "macos-14-arm64"]' || '"macos-14"') }} | |
| target: aarch64-apple-darwin | |
| suffix: macos-aarch64-${{ github.ref_name }} | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} | |
| target: x86_64-pc-windows-msvc | |
| suffix: windows-x86_64-skylake-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=skylake" | |
| - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} | |
| target: x86_64-pc-windows-msvc | |
| suffix: windows-x86_64-v2-${{ github.ref_name }} | |
| rustflags: "-C target-cpu=x86-64-v2" | |
| fail-fast: false | |
| runs-on: ${{ matrix.build.os }} | |
| env: | |
| PRODUCTION_TARGET: target/${{ matrix.build.target }}/production | |
| RUSTFLAGS: ${{ matrix.build.rustflags }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| # On macOS, we need a proper Clang version, not Apple's custom version without wasm32 support | |
| - name: Install LLVM and Clang for macOS | |
| uses: KyleMayes/install-llvm-action@dec985c8d7b46a2f363ea1a78f660c946a3349ea # v2.0.1 | |
| with: | |
| env: true | |
| version: 17 | |
| if: runner.os == 'macOS' | |
| # Because macOS, see https://andreasfertig.blog/2021/02/clang-and-gcc-on-macos-catalina-finding-the-include-paths/ | |
| - name: Configure C compiler macOS | |
| run: | | |
| echo "SDKROOT=$(xcrun --show-sdk-path)" >> $GITHUB_ENV | |
| if: runner.os == 'macOS' | |
| - name: Install glibtoolize (macOS) | |
| run: brew install libtool | |
| if: runner.os == 'macOS' | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Needed for hwloc | |
| - name: Install automake (macOS) | |
| run: brew install automake | |
| if: runner.os == 'macOS' | |
| # Workaround to resolve link error with C:\msys64\mingw64\bin\libclang.dll | |
| - name: Remove msys64 | |
| run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse | |
| # Doesn't exist on self-hosted runners | |
| if: matrix.os == 'windows-2022' | |
| - name: AArch64 cross-compile packages | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| g++-aarch64-linux-gnu \ | |
| gcc-aarch64-linux-gnu \ | |
| libc6-dev-arm64-cross | |
| echo "PKG_CONFIG_ALLOW_CROSS=true" >> $GITHUB_ENV | |
| if: matrix.build.target == 'aarch64-unknown-linux-gnu' | |
| - name: CUDA toolchain | |
| uses: Jimver/cuda-toolkit@9b295696791d75d658d8de64c4a85097ad8abeaf # v0.2.16 | |
| with: | |
| cuda: '12.4.1' | |
| method: network | |
| sub-packages: '["nvcc", "cudart"]' | |
| if: runner.os == 'Linux' || runner.os == 'Windows' | |
| # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it | |
| # - name: Configure ROCm cache (Windows) | |
| # uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| # id: rocm-cache | |
| # with: | |
| # path: C:\Program Files\AMD\ROCm | |
| # key: ${{ runner.os }}-rocm | |
| # if: runner.os == 'Windows' | |
| - name: ROCm toolchain | |
| run: | | |
| ROCM_VERSION=6.2.2 | |
| sudo mkdir -p --mode=0755 /etc/apt/keyrings | |
| curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null | |
| echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION focal main" | sudo tee /etc/apt/sources.list.d/rocm.list > /dev/null | |
| echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" | sudo tee /etc/apt/preferences.d/rocm-pin-600 > /dev/null | |
| sudo apt-get update | |
| DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends rocm-hip-runtime-dev | |
| echo "/opt/rocm/lib" | sudo tee /etc/ld.so.conf.d/rocm.conf > /dev/null | |
| sudo ldconfig | |
| # TODO: ROCm packages are only available for x86-64 for now | |
| if: runner.os == 'Linux' && startsWith(matrix.build.target, 'x86_64') | |
| # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it | |
| # - name: ROCm toolchain | |
| # run: | | |
| # $ErrorActionPreference = "Stop" | |
| # Invoke-WebRequest -Uri https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe -OutFile "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" | |
| # Start-Process "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" -ArgumentList '-install' -NoNewWindow -Wait | |
| # Remove-Item "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" | |
| # if: runner.os == 'Windows' && steps.rocm-cache.outputs.cache-hit != 'true' | |
| # | |
| # - name: ROCm toolchain environment (Windows) | |
| # run: | | |
| # Add-Content $env:GITHUB_PATH "C:\Program Files\AMD\ROCm\6.1\bin" | |
| # if: runner.os == 'Windows' | |
| - name: Configure cache | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo- | |
| - name: Build farmer | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer | |
| if: runner.os == 'macOS' || !startsWith(matrix.build.target, 'x86_64') | |
| # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it | |
| # # ROCm can't be enabled together with CUDA for now | |
| # - name: Build farmer (ROCm, Windows) | |
| # env: | |
| # NVCC: off | |
| # HIPCC: hipcc.bin.exe | |
| # run: | | |
| # cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features rocm | |
| # move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe | |
| # # TODO: ROCm packages are only available for x86-64 for now | |
| # if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64') | |
| # ROCm can't be enabled together with CUDA for now | |
| - name: Build farmer (ROCm, Ubuntu) | |
| env: | |
| NVCC: off | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features rocm | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm | |
| # TODO: ROCm packages are only available for x86-64 for now | |
| if: runner.os == 'Linux' && startsWith(matrix.build.target, 'x86_64') | |
| - name: Build farmer | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features cuda | |
| # TODO: We don't configure CUDA for cross-compilation purposes, hence only x86-64 for now | |
| if: (runner.os == 'Linux' || runner.os == 'Windows') && startsWith(matrix.build.target, 'x86_64') | |
| - name: Build node | |
| run: | | |
| cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-node | |
| - name: Sign Application (macOS) | |
| run: | | |
| echo "Importing certificate" | |
| echo "${{ secrets.MACOS_CERTIFICATE }}" | base64 --decode > certificate.p12 | |
| security create-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| security default-keychain -s build.keychain | |
| security unlock-keychain -p "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| security import certificate.p12 -k build.keychain -P "${{ secrets.MACOS_CERTIFICATE_PW }}" -T /usr/bin/codesign | |
| security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${{ secrets.MACOS_CERTIFICATE_PW }}" build.keychain | |
| echo "Signing farmer" | |
| codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-farmer | |
| echo "Signing node" | |
| codesign --force --options=runtime --entitlements .github/workflows/Entitlements.plist -s "${{ secrets.MACOS_IDENTITY }}" --timestamp ${{ env.PRODUCTION_TARGET }}/subspace-node | |
| echo "Creating a ZIP archive" | |
| mkdir -p ${{ env.PRODUCTION_TARGET }}/macos-binaries | |
| cp ${{ env.PRODUCTION_TARGET }}/subspace-farmer ${{ env.PRODUCTION_TARGET }}/subspace-node ${{ env.PRODUCTION_TARGET }}/macos-binaries | |
| ditto -c -k --rsrc ${{ env.PRODUCTION_TARGET }}/macos-binaries subspace-binaries.zip | |
| echo "Notarizing ZIP archive file" | |
| xcrun notarytool submit subspace-binaries.zip --apple-id "${{ secrets.MACOS_APPLE_ID }}" --password "${{ secrets.MACOS_APP_PW }}" --team-id "${{ secrets.MACOS_TEAM_ID }}" --wait | |
| # stapling does not work for .zip archives only .app bundles and .dmg files. Commenting this for now! | |
| # echo "Stapling notarization to ZIP file" | |
| # xcrun stapler staple subspace-binaries.zip | |
| echo "Done!" | |
| # Allow code signing to fail on non-release builds and in non-subspace repos (forks) | |
| continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }} | |
| if: runner.os == 'macOS' | |
| - name: Sign Application (Windows) | |
| run: | | |
| AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe" | |
| # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it | |
| # AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe" | |
| AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe" | |
| # Allow code signing to fail on non-release builds and in non-subspace repos (forks) | |
| continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }} | |
| if: runner.os == 'Windows' | |
| - name: Prepare executables for uploading (Ubuntu) | |
| run: | | |
| mkdir executables | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }} | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }} | |
| if: runner.os == 'Linux' | |
| - name: Prepare executables for uploading (Ubuntu, ROCm) | |
| run: | | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm executables/subspace-farmer-rocm-${{ matrix.build.suffix }} | |
| # TODO: ROCm packages are only available for x86-64 for now | |
| if: runner.os == 'Linux' && startsWith(matrix.build.target, 'x86_64') | |
| - name: Prepare executables for uploading (macOS) | |
| run: | | |
| mkdir executables | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-farmer executables/subspace-farmer-${{ matrix.build.suffix }} | |
| mv ${{ env.PRODUCTION_TARGET }}/subspace-node executables/subspace-node-${{ matrix.build.suffix }} | |
| # Zip it so that signature is not lost | |
| ditto -c -k --rsrc executables/subspace-farmer-${{ matrix.build.suffix }} executables/subspace-farmer-${{ matrix.build.suffix }}.zip | |
| ditto -c -k --rsrc executables/subspace-node-${{ matrix.build.suffix }} executables/subspace-node-${{ matrix.build.suffix }}.zip | |
| rm executables/subspace-farmer-${{ matrix.build.suffix }} | |
| rm executables/subspace-node-${{ matrix.build.suffix }} | |
| if: runner.os == 'macOS' | |
| - name: Prepare executables for uploading (Windows) | |
| run: | | |
| mkdir executables | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ matrix.build.suffix }}.exe | |
| # TODO: ROCm compilation doesn't work in CI right now, good luck fixing it | |
| # move ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe executables/subspace-farmer-rocm-${{ matrix.build.suffix }}.exe | |
| move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ matrix.build.suffix }}.exe | |
| if: runner.os == 'Windows' | |
| - name: Upload node and farmer executables to artifacts | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.1.3 | |
| with: | |
| name: executables-${{ matrix.build.suffix }} | |
| path: | | |
| executables/* | |
| if-no-files-found: error | |
| - name: Upload node and farmer executables to assets | |
| uses: alexellis/upload-assets@13926a61cdb2cb35f5fdef1c06b8b591523236d3 # 0.4.1 | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| with: | |
| asset_paths: '["executables/*"]' | |
| if: github.event_name == 'push' && github.ref_type == 'tag' |