Cross build packages #829
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cross build packages | |
| concurrency: | |
| group: build-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| schedule: | |
| - cron: '0 */6 * * *' | |
| push: | |
| branches: | |
| - 'ci/**' | |
| tags: | |
| - v* | |
| - e* | |
| workflow_dispatch: | |
| inputs: | |
| branch_or_tag: | |
| required: false | |
| profile: | |
| required: false | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-22.04 | |
| if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule' | |
| container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04 | |
| outputs: | |
| BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} | |
| IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} | |
| VERSION: ${{ steps.get_profile.outputs.VERSION }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used | |
| path: source | |
| fetch-depth: 0 | |
| - name: Get profile to build | |
| id: get_profile | |
| run: | | |
| cd source | |
| git config --global --add safe.directory "$(pwd)" | |
| tag=${{ github.ref }} | |
| if git describe --tags --match "[v|e]*" --exact; then | |
| echo "WARN: This is an exact git tag, will publish release" | |
| is_exact_tag='true' | |
| else | |
| echo "WARN: This is NOT an exact git tag, will not publish release" | |
| is_exact_tag='false' | |
| fi | |
| echo "IS_EXACT_TAG=${is_exact_tag}" >> $GITHUB_OUTPUT | |
| case $tag in | |
| refs/tags/v*) | |
| PROFILE='emqx' | |
| ;; | |
| refs/tags/e*) | |
| PROFILE=emqx-enterprise | |
| ;; | |
| *) | |
| PROFILE=${{ github.event.inputs.profile }} | |
| case "$PROFILE" in | |
| emqx) | |
| true | |
| ;; | |
| emqx-enterprise) | |
| true | |
| ;; | |
| *) | |
| # maybe triggered from schedule | |
| echo "WARN: \"$PROFILE\" is not a valid profile." | |
| echo "building the default profile 'emqx' instead" | |
| PROFILE='emqx' | |
| ;; | |
| esac | |
| ;; | |
| esac | |
| echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT | |
| echo "VERSION=$(./pkg-vsn.sh $PROFILE)" >> $GITHUB_OUTPUT | |
| - name: get_all_deps | |
| run: | | |
| make -C source deps-all | |
| zip -ryq source.zip source/* source/.[^.]* | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: source | |
| path: source.zip | |
| windows: | |
| runs-on: windows-2019 | |
| if: startsWith(github.ref_name, 'v') | |
| needs: prepare | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| profile: # for now only CE for windows | |
| - emqx | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: source | |
| path: . | |
| - name: unzip source code | |
| run: Expand-Archive -Path source.zip -DestinationPath ./ | |
| - uses: ilammy/[email protected] | |
| - uses: erlef/[email protected] | |
| with: | |
| otp-version: 24.3.4.6 | |
| - name: build | |
| env: | |
| PYTHON: python | |
| DIAGNOSTIC: 1 | |
| working-directory: source | |
| run: | | |
| # ensure crypto app (openssl) | |
| erl -eval "erlang:display(crypto:info_lib())" -s init stop | |
| make ${{ matrix.profile }}-tgz | |
| - name: run emqx | |
| timeout-minutes: 5 | |
| working-directory: source | |
| run: | | |
| ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start | |
| Start-Sleep -s 5 | |
| echo "EMQX started" | |
| ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop | |
| echo "EMQX stopped" | |
| ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install | |
| echo "EMQX installed" | |
| ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall | |
| echo "EMQX uninstalled" | |
| - uses: actions/upload-artifact@v3 | |
| if: success() | |
| with: | |
| name: ${{ matrix.profile }} | |
| path: source/_packages/${{ matrix.profile }}/ | |
| - name: Send notification to Slack | |
| uses: slackapi/[email protected] | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| with: | |
| payload: | | |
| {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} | |
| mac: | |
| needs: prepare | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| profile: | |
| - ${{ needs.prepare.outputs.BUILD_PROFILE }} | |
| otp: | |
| - 24.3.4.2-3 | |
| os: | |
| - macos-11 | |
| - macos-12 | |
| - macos-12-arm64 | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: emqx/[email protected] | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: source | |
| path: . | |
| - name: unzip source code | |
| run: | | |
| ln -s . source | |
| unzip -o -q source.zip | |
| rm source source.zip | |
| - uses: ./.github/actions/package-macos | |
| with: | |
| profile: ${{ matrix.profile }} | |
| otp: ${{ matrix.otp }} | |
| os: ${{ matrix.os }} | |
| apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} | |
| apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} | |
| apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} | |
| apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} | |
| - uses: actions/upload-artifact@v3 | |
| if: success() | |
| with: | |
| name: ${{ matrix.profile }} | |
| path: _packages/${{ matrix.profile }}/ | |
| - name: Send notification to Slack | |
| uses: slackapi/[email protected] | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| with: | |
| payload: | | |
| {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} | |
| linux: | |
| needs: prepare | |
| runs-on: ${{ matrix.build_machine }} | |
| container: | |
| image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| profile: | |
| - ${{ needs.prepare.outputs.BUILD_PROFILE }} | |
| otp: | |
| - 24.3.4.2-3 | |
| arch: | |
| - amd64 | |
| - arm64 | |
| os: | |
| - ubuntu22.04 | |
| - ubuntu20.04 | |
| - ubuntu18.04 | |
| - debian11 | |
| - debian10 | |
| - el9 | |
| - el8 | |
| - el7 | |
| - amzn2 | |
| build_machine: | |
| - aws-arm64 | |
| - ubuntu-22.04 | |
| builder: | |
| - 5.0-34 | |
| elixir: | |
| - 1.13.4 | |
| exclude: | |
| - arch: arm64 | |
| build_machine: ubuntu-22.04 | |
| - arch: amd64 | |
| build_machine: aws-arm64 | |
| include: | |
| - profile: emqx | |
| otp: 25.1.2-3 | |
| arch: amd64 | |
| os: ubuntu22.04 | |
| build_machine: ubuntu-22.04 | |
| builder: 5.0-34 | |
| elixir: 1.13.4 | |
| release_with: elixir | |
| - profile: emqx | |
| otp: 25.1.2-3 | |
| arch: amd64 | |
| os: amzn2 | |
| build_machine: ubuntu-22.04 | |
| builder: 5.0-34 | |
| elixir: 1.13.4 | |
| release_with: elixir | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: AutoModality/action-clean@v1 | |
| if: matrix.build_machine == 'aws-arm64' | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: source | |
| path: . | |
| - name: unzip source code | |
| run: unzip -q source.zip | |
| - name: tmp fix for el9 | |
| if: matrix.os == 'el9' | |
| run: | | |
| set -eu | |
| dnf install -y krb5-devel | |
| - name: build emqx packages | |
| working-directory: source | |
| env: | |
| BUILDER: ${{ matrix.builder }} | |
| ELIXIR: ${{ matrix.elixir }} | |
| OTP: ${{ matrix.otp }} | |
| PROFILE: ${{ matrix.profile }} | |
| ARCH: ${{ matrix.arch }} | |
| SYSTEM: ${{ matrix.os }} | |
| run: | | |
| set -eu | |
| git config --global --add safe.directory "/__w/emqx/emqx" | |
| # Align path for CMake caches | |
| if [ ! "$PWD" = "/emqx" ]; then | |
| ln -s $PWD /emqx | |
| cd /emqx | |
| fi | |
| echo "pwd is $PWD" | |
| PKGTYPES="tgz pkg" | |
| IS_ELIXIR="no" | |
| if [ ${{ matrix.release_with }} == 'elixir' ]; then | |
| PKGTYPES="tgz" | |
| # set Elixir build flag | |
| IS_ELIXIR="yes" | |
| fi | |
| for PKGTYPE in ${PKGTYPES}; | |
| do | |
| ./scripts/buildx.sh \ | |
| --profile "${PROFILE}" \ | |
| --pkgtype "${PKGTYPE}" \ | |
| --arch "${ARCH}" \ | |
| --elixir "${IS_ELIXIR}" \ | |
| --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" | |
| done | |
| - uses: actions/upload-artifact@v3 | |
| if: success() | |
| with: | |
| name: ${{ matrix.profile }} | |
| path: source/_packages/${{ matrix.profile }}/ | |
| - name: Send notification to Slack | |
| uses: slackapi/[email protected] | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| with: | |
| payload: | | |
| {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} | |
| publish_artifacts: | |
| runs-on: ubuntu-22.04 | |
| needs: [prepare, mac, linux] | |
| if: needs.prepare.outputs.IS_EXACT_TAG && github.event_name != 'schedule' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| profile: | |
| - ${{ needs.prepare.outputs.BUILD_PROFILE }} | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ matrix.profile }} | |
| path: packages/${{ matrix.profile }} | |
| - name: install dos2unix | |
| run: sudo apt-get update && sudo apt install -y dos2unix | |
| - name: get packages | |
| run: | | |
| set -e -u | |
| cd packages/${{ matrix.profile }} | |
| # fix the .sha256 file format | |
| for var in $(ls | grep emqx | grep -v sha256); do | |
| dos2unix $var.sha256 | |
| echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 | |
| done | |
| cd - | |
| - uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| - name: upload to aws s3 | |
| env: | |
| PROFILE: ${{ matrix.profile }} | |
| run: | | |
| set -e -u | |
| if [ $PROFILE = 'emqx' ]; then | |
| s3dir='emqx-ce' | |
| elif [ $PROFILE = 'emqx-enterprise' ]; then | |
| s3dir='emqx-ee' | |
| else | |
| echo "unknown profile $PROFILE" | |
| exit 1 | |
| fi | |
| aws s3 cp --recursive packages/$PROFILE s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} | |
| aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" | |
| - name: Push to packagecloud.io | |
| env: | |
| PROFILE: ${{ matrix.profile }} | |
| VERSION: ${{ needs.prepare.outputs.VERSION }} | |
| PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} | |
| run: | | |
| set -eu | |
| REPO=$PROFILE | |
| if [ $PROFILE = 'emqx-enterprise' ]; then | |
| REPO='emqx-enterprise5' | |
| fi | |
| function push() { | |
| docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/$2:/w/$2 -w /w ghcr.io/emqx/package_cloud push emqx/$REPO/$1 $2 | |
| } | |
| push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-amd64.deb" | |
| push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-arm64.deb" | |
| push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-amd64.deb" | |
| push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-arm64.deb" | |
| push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-amd64.deb" | |
| push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-arm64.deb" | |
| push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-amd64.deb" | |
| push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" | |
| push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-amd64.deb" | |
| push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-arm64.deb" | |
| push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-amd64.rpm" | |
| push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-arm64.rpm" | |
| push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-amd64.rpm" | |
| push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-arm64.rpm" | |
| push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-amd64.rpm" | |
| push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-arm64.rpm" | |
| push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-amd64.rpm" | |
| push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-arm64.rpm" |