[WIP] wheel distro #9
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AMD AIE Distro | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| DEBUG_ENABLED: | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| type: boolean | |
| required: false | |
| default: false | |
| DEBUG_OS: | |
| description: 'which runner os to run the tmate action in (if the tmate action is run)' | |
| type: string | |
| default: 'windows-2019' | |
| required: false | |
| DEBUG_ARCH: | |
| description: 'which runner arch to run the tmate action in (if the tmate action is run)' | |
| type: string | |
| default: 'x86_64' | |
| required: false | |
| DEBUG_DETACHED: | |
| description: 'whether to launch tmate in detached mode (if the tmate action is run)' | |
| type: boolean | |
| required: false | |
| default: true | |
| LLVM_AIE_COMMIT: | |
| description: 'llvm-aie commit to build' | |
| type: string | |
| required: false | |
| default: '' | |
| APPLY_PATCHES: | |
| description: 'whether to apply patches to source' | |
| type: string | |
| required: false | |
| default: 'true' | |
| RUN_TESTS: | |
| description: 'whether to run llvm tests after build' | |
| type: string | |
| required: false | |
| default: 'false' | |
| pull_request: | |
| release: | |
| types: | |
| - published | |
| schedule: | |
| # At minute 0 past hour 1. (see https://crontab.guru) | |
| # this is the actual nightly | |
| - cron: '00 01 * * *' | |
| # At minute 0 past every 4th hour. (see https://crontab.guru) | |
| # this job is to keep the ccache cache warm | |
| - cron: '0 */4 * * *' | |
| jobs: | |
| get_llvm_aie_project_commit: | |
| name: Get latest LLVM-AIE commit | |
| runs-on: ubuntu-latest | |
| outputs: | |
| LLVM_AIE_PROJECT_COMMIT: ${{ steps.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }} | |
| DATETIME: ${{ steps.get_llvm_aie_project_commit.outputs.DATETIME }} | |
| steps: | |
| # have to do this until repo is public | |
| - name: Checkout actions | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.head_ref || github.ref_name }} | |
| # check out anything | |
| sparse-checkout: .github/workflows/amd_aie_releases | |
| sparse-checkout-cone-mode: false | |
| - name: Get llvm-aie commit | |
| id: get_llvm_aie_project_commit | |
| run: | | |
| if [ x"${{ inputs.LLVM_AIE_COMMIT }}" == x"" ]; then | |
| LLVM_AIE_PROJECT_COMMIT=$(git rev-parse ${{ github.head_ref || github.ref_name }}) | |
| LLVM_AIE_PROJECT_COMMIT=${LLVM_AIE_PROJECT_COMMIT:0:8} | |
| else | |
| LLVM_AIE_PROJECT_COMMIT="${{ inputs.LLVM_AIE_COMMIT }}" | |
| fi | |
| echo "LLVM_AIE_PROJECT_COMMIT=${LLVM_AIE_PROJECT_COMMIT}" | tee -a $GITHUB_OUTPUT | |
| DATETIME=$(date +"%Y%m%d%H") | |
| echo "DATETIME=${DATETIME}" | tee -a $GITHUB_OUTPUT | |
| build: | |
| needs: get_llvm_aie_project_commit | |
| name: ${{ matrix.OS }} ${{ matrix.ARCH }} | |
| continue-on-error: true | |
| runs-on: ${{ matrix.OS }} | |
| outputs: | |
| LLVM_AIE_WHEEL_VERSION: ${{ steps.get_wheel_version.outputs.LLVM_AIE_WHEEL_VERSION }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - OS: ubuntu-20.04 | |
| ARCH: x86_64 | |
| - OS: ubuntu-20.04 | |
| ARCH: aarch64 | |
| - OS: windows-2019 | |
| ARCH: AMD64 | |
| - OS: macos-12 | |
| ARCH: x86_64 | |
| - OS: macos-12 | |
| ARCH: arm64 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: makslevental/mlir-wheels/actions/setup_base@54495f0f8fea29747d85daad4da0f815d8b3d5a4 | |
| id: setup_base | |
| with: | |
| # optional | |
| DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }} | |
| DEBUG_OS: ${{ inputs.DEBUG_OS }} | |
| DEBUG_ARCH: ${{ inputs.DEBUG_ARCH }} | |
| DEBUG_DETACHED: ${{ inputs.DEBUG_DETACHED }} | |
| # required | |
| MATRIX_OS: ${{ matrix.OS }} | |
| MATRIX_ARCH: ${{ matrix.ARCH }} | |
| - uses: makslevental/mlir-wheels/actions/setup_ccache@54495f0f8fea29747d85daad4da0f815d8b3d5a4 | |
| id: setup_ccache | |
| with: | |
| MATRIX_OS: ${{ matrix.OS }} | |
| MATRIX_ARCH: ${{ matrix.ARCH }} | |
| WORKSPACE_ROOT: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| - name: Checkout actions | |
| uses: actions/checkout@v3 | |
| with: | |
| sparse-checkout: .github/workflows/amd_aie_releases | |
| # Turning off cone mode ensures that files in the project root are not included during checkout | |
| sparse-checkout-cone-mode: false | |
| # This step is needed because action/checkout does not support paths. | |
| - name: Move release utils app to workspace_root | |
| run: | | |
| shopt -s dotglob | |
| WORKSPACE_ROOT="${{ steps.setup_base.outputs.WORKSPACE_ROOT }}" | |
| if [[ ${{ matrix.OS }} == *"windows"* ]]; then | |
| WORKSPACE_ROOT=$(echo "/$WORKSPACE_ROOT" | sed -e 's/\\/\//g' -e 's/://') | |
| fi | |
| ls -lah | |
| mv .github/workflows/amd_aie_releases/* "$WORKSPACE_ROOT" | |
| ls -lah "$WORKSPACE_ROOT" | |
| - name: Get LLVM-AIE | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| LLVM_AIE_PROJECT_COMMIT=${{ needs.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }} | |
| gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| /repos/Xilinx/llvm-aie/zipball/$LLVM_AIE_PROJECT_COMMIT > llvm-aie.zip | |
| unzip -q llvm-aie.zip | |
| rm -rf llvm-aie.zip | |
| mv llvm-aie-$LLVM_AIE_PROJECT_COMMIT llvm-aie | |
| # build | |
| - name: cibuildwheel | |
| # you need the parens here because of the != and something about yaml??? | |
| if: (!contains(matrix.OS, 'ubuntu') || matrix.ARCH != 'aarch64') | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} \ | |
| RUN_TESTS=${{ inputs.RUN_TESTS == '' && 'false' || inputs.RUN_TESTS }} \ | |
| CIBW_ARCHS=${{ matrix.ARCH }} \ | |
| CMAKE_GENERATOR=Ninja \ | |
| DATETIME=${{ needs.get_llvm_aie_project_commit.outputs.DATETIME }} \ | |
| HOST_CCACHE_DIR=${{ steps.setup_ccache.outputs.HOST_CCACHE_DIR }} \ | |
| LLVM_AIE_PROJECT_COMMIT=${{ needs.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }} \ | |
| MATRIX_OS=${{ matrix.OS }} \ | |
| PARALLEL_LEVEL=2 \ | |
| cibuildwheel --output-dir wheelhouse | |
| - name: build aarch ubuntu wheel | |
| if: contains(matrix.OS, 'ubuntu') && matrix.ARCH == 'aarch64' | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| export APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} | |
| ./scripts/apply_patches.sh | |
| CIBW_ARCHS=${{ matrix.ARCH }} \ | |
| CMAKE_GENERATOR=Ninja \ | |
| DATETIME=${{ needs.get_llvm_aie_project_commit.outputs.DATETIME }} \ | |
| LLVM_AIE_PROJECT_COMMIT=${{ needs.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }} \ | |
| MATRIX_OS=${{ matrix.OS }} \ | |
| PARALLEL_LEVEL=2 \ | |
| PIP_FIND_LINKS="https://github.com/makslevental/llvm-aie-wheels/releases/expanded_assets/latest" \ | |
| pip wheel . -v -w wheelhouse | |
| - name: Download cache from container ubuntu | |
| if: contains(matrix.OS, 'ubuntu') && contains(matrix.ARCH, 'x86_64') && (success() || failure()) | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ccache -s | |
| HOST_CCACHE_DIR="$(ccache --get-config cache_dir)" | |
| rm -rf $HOST_CCACHE_DIR | |
| mv ./wheelhouse/.ccache $HOST_CCACHE_DIR | |
| ls -la $HOST_CCACHE_DIR | |
| ccache -s | |
| - name: Reset datetime ccache | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ccache --print-stats | |
| HOST_CCACHE_DIR="$(ccache --get-config cache_dir)" | |
| find $HOST_CCACHE_DIR -exec touch -a -m -t "${{ needs.get_llvm_aie_project_commit.outputs.DATETIME }}00" {} \; | |
| - name: Clean llvm-aie | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| rm -rf llvm-aie | |
| rm -rf build | |
| - name: Docker prune | |
| if: contains(matrix.OS, 'ubuntu') | |
| run: | | |
| docker system prune -a -f | |
| - name: Get wheel version | |
| id: get_wheel_version | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| pip install pkginfo | |
| WHL=$(ls wheelhouse/llvm-aie-*whl) | |
| echo "LLVM_AIE_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT | |
| - name: rename non-windows | |
| if: contains(matrix.OS, 'ubuntu') || contains(matrix.OS, 'macos') | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| rename 's/cp310-cp310/py3-none/' wheelhouse/llvm-aie-*whl | |
| rename 's/cp311-cp311/py3-none/' wheelhouse/llvm-aie-*whl | |
| if [[ ${{ matrix.OS }} == *"ubuntu"* ]] && [ x"${{ matrix.ARCH }}" == x"aarch64" ]; then | |
| rename 's/x86_64/aarch64/' wheelhouse/llvm-aie-*whl | |
| fi | |
| - name: rename windows | |
| if: contains(matrix.OS, 'windows') | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| ls wheelhouse/llvm-aie-*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' } | |
| ls wheelhouse/llvm-aie-*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' } | |
| - name: Build native_tools wheel | |
| working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }} | |
| id: build_native_tools_wheel | |
| run: | | |
| for TOOL in "llvm-tblgen" "llvm-aie-tblgen" "llvm-aie-linalg-ods-yaml-gen" "llvm-aie-pdll" "llvm-config" "FileCheck"; do | |
| if [[ ${{ matrix.OS }} == *"windows"* ]]; then | |
| TOOL="$TOOL.exe" | |
| fi | |
| unzip -j wheelhouse/llvm-aie-*whl "llvm_aie/bin/$TOOL" -d native_tools/ | |
| done | |
| if [[ ${{ matrix.OS }} == *"ubuntu"* ]]; then | |
| PLAT="linux" | |
| elif [[ ${{ matrix.OS }} == *"macos"* ]]; then | |
| PLAT="macosx_12_0" | |
| elif [[ ${{ matrix.OS }} == *"windows"* ]]; then | |
| PLAT="win" | |
| fi | |
| PLAT=${PLAT}_$(echo ${{ matrix.ARCH }} | tr '[:upper:]' '[:lower:]') | |
| pushd native_tools | |
| LLVM_AIE_WHEEL_VERSION=${{ steps.get_wheel_version.outputs.LLVM_AIE_WHEEL_VERSION }} \ | |
| python setup.py bdist_wheel --dist-dir ../wheelhouse --plat $PLAT | |
| popd | |
| # done | |
| - name: Upload wheels | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl | |
| name: build_artifact | |
| upload_distro_wheels: | |
| if: github.event.schedule != '0 */4 * * *' | |
| needs: build | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| # unpacks default artifact into dist/ | |
| # if `name: artifact` is omitted, the action will create extra parent dir | |
| name: build_artifact | |
| path: dist | |
| - name: Release current commit | |
| uses: ncipollo/[email protected] | |
| with: | |
| artifacts: "dist/*.whl,dist/*.tar.xz" | |
| token: "${{ secrets.GITHUB_TOKEN }}" | |
| tag: "nightly" | |
| name: "nightly" | |
| removeArtifacts: false | |
| allowUpdates: true | |
| replacesArtifacts: true | |
| makeLatest: true |