[WIP] wheel distro

.github/workflows/amd-aie-clean-nightly-releases.yml

@@ -0,0 +1,35 @@
+name: AMD AIE Clean up old releases
+
+on:
+  workflow_dispatch:
+    branches:
+      - main
+  schedule:
+    # At minute 0 past hour 6. (see https://crontab.guru)
+    - cron: '00 06 * * *'
+
+jobs:
+
+  delete_releases:
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Delete from here
+        shell: bash
+        run: |
+          
+          pip install PyGithub
+          GITHUB_TOKEN=${{ secrets.DELETE_RELEASES_TOKEN }} python scripts/gh_releases.py

.github/workflows/amd-aie-distro.yml

@@ -0,0 +1,343 @@
+name: AMD AIE Distro
+
+on:
+  workflow_dispatch:
+    inputs:
+      DEBUG_ENABLED:
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        type: boolean
+        required: false
+        default: false
+      DEBUG_OS:
+        description: 'which runner os to run the tmate action in (if the tmate action is run)'
+        type: string
+        default: 'windows-2019'
+        required: false
+      DEBUG_ARCH:
+        description: 'which runner arch to run the tmate action in (if the tmate action is run)'
+        type: string
+        default: 'x86_64'
+        required: false
+      DEBUG_DETACHED:
+        description: 'whether to launch tmate in detached mode (if the tmate action is run)'
+        type: boolean
+        required: false
+        default: true
+      LLVM_AIE_COMMIT:
+        description: 'llvm-aie commit to build'
+        type: string
+        required: false
+        default: ''
+      APPLY_PATCHES:
+        description: 'whether to apply patches to source'
+        type: string
+        required: false
+        default: 'true'
+      RUN_TESTS:
+        description: 'whether to run llvm tests after build'
+        type: string
+        required: false
+        default: 'false'
+
+  pull_request:
+  release:
+    types:
+      - published
+
+  schedule:
+    # At minute 0 past hour 1. (see https://crontab.guru)
+    # this is the actual nightly
+    - cron: '00 01 * * *'
+      # At minute 0 past every 4th hour. (see https://crontab.guru)
+      # this job is to keep the ccache cache warm
+    - cron: '0 */4 * * *'
+
+jobs:
+
+  get_llvm_aie_project_commit:
+
+    name: Get latest LLVM-AIE commit
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      LLVM_AIE_PROJECT_COMMIT: ${{ steps.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }}
+      DATETIME: ${{ steps.get_llvm_aie_project_commit.outputs.DATETIME }}
+
+    steps:
+      # have to do this until repo is public
+      - name: Checkout actions
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+          # check out anything
+          sparse-checkout: .github/workflows/amd_aie_releases
+          sparse-checkout-cone-mode: false
+
+      - name: Get llvm-aie commit
+        id: get_llvm_aie_project_commit
+        run: |
+          
+          if [ x"${{ inputs.LLVM_AIE_COMMIT }}" == x"" ]; then
+            LLVM_AIE_PROJECT_COMMIT=$(git rev-parse ${{ github.head_ref || github.ref_name }})
+            LLVM_AIE_PROJECT_COMMIT=${LLVM_AIE_PROJECT_COMMIT:0:8}
+          else
+            LLVM_AIE_PROJECT_COMMIT="${{ inputs.LLVM_AIE_COMMIT }}"
+          fi
+          echo "LLVM_AIE_PROJECT_COMMIT=${LLVM_AIE_PROJECT_COMMIT}" | tee -a $GITHUB_OUTPUT
+          DATETIME=$(date +"%Y%m%d%H")
+          echo "DATETIME=${DATETIME}" | tee -a $GITHUB_OUTPUT
+
+  build:
+
+    needs: get_llvm_aie_project_commit
+
+    name: ${{ matrix.OS }} ${{ matrix.ARCH }}
+
+    continue-on-error: true
+
+    runs-on: ${{ matrix.OS }}
+
+    outputs:
+      LLVM_AIE_WHEEL_VERSION: ${{ steps.get_wheel_version.outputs.LLVM_AIE_WHEEL_VERSION }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - OS: ubuntu-20.04
+            ARCH: x86_64
+
+          - OS: ubuntu-20.04
+            ARCH: aarch64
+
+          - OS: windows-2019
+            ARCH: AMD64
+
+          - OS: macos-12
+            ARCH: x86_64
+
+          - OS: macos-12
+            ARCH: arm64
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+
+    - uses: makslevental/mlir-wheels/actions/setup_base@54495f0f8fea29747d85daad4da0f815d8b3d5a4
+      id: setup_base
+      with:
+        # optional
+        DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }}
+        DEBUG_OS: ${{ inputs.DEBUG_OS }}
+        DEBUG_ARCH: ${{ inputs.DEBUG_ARCH }}
+        DEBUG_DETACHED: ${{ inputs.DEBUG_DETACHED }}
+        # required
+        MATRIX_OS: ${{ matrix.OS }}
+        MATRIX_ARCH: ${{ matrix.ARCH }}
+
+    - uses: makslevental/mlir-wheels/actions/setup_ccache@54495f0f8fea29747d85daad4da0f815d8b3d5a4
+      id: setup_ccache
+      with:
+        MATRIX_OS: ${{ matrix.OS }}
+        MATRIX_ARCH: ${{ matrix.ARCH }}
+        WORKSPACE_ROOT: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+
+    - name: Checkout actions
+      uses: actions/checkout@v3
+      with:
+        sparse-checkout: .github/workflows/amd_aie_releases
+        # Turning off cone mode ensures that files in the project root are not included during checkout
+        sparse-checkout-cone-mode: false
+
+    # This step is needed because action/checkout does not support paths.
+    - name: Move release utils app to workspace_root
+      run: |
+        
+        shopt -s dotglob
+        
+        WORKSPACE_ROOT="${{ steps.setup_base.outputs.WORKSPACE_ROOT }}"
+        if [[ ${{ matrix.OS }} == *"windows"* ]]; then
+          WORKSPACE_ROOT=$(echo "/$WORKSPACE_ROOT" | sed -e 's/\\/\//g' -e 's/://')
+        fi
+        
+        ls -lah
+        mv .github/workflows/amd_aie_releases/* "$WORKSPACE_ROOT"
+        ls -lah "$WORKSPACE_ROOT"
+
+    - name: Get LLVM-AIE
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        
+        LLVM_AIE_PROJECT_COMMIT=${{ needs.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }}
+        gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
+          /repos/Xilinx/llvm-aie/zipball/$LLVM_AIE_PROJECT_COMMIT > llvm-aie.zip
+        unzip -q llvm-aie.zip
+        rm -rf llvm-aie.zip
+        mv llvm-aie-$LLVM_AIE_PROJECT_COMMIT llvm-aie
+
+    # build
+
+    - name: cibuildwheel
+      # you need the parens here because of the != and something about yaml???
+      if: (!contains(matrix.OS, 'ubuntu') || matrix.ARCH != 'aarch64')
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        
+        APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }} \
+        RUN_TESTS=${{ inputs.RUN_TESTS == '' && 'false' || inputs.RUN_TESTS }} \
+        CIBW_ARCHS=${{ matrix.ARCH }} \
+        CMAKE_GENERATOR=Ninja \
+        DATETIME=${{ needs.get_llvm_aie_project_commit.outputs.DATETIME }} \
+        HOST_CCACHE_DIR=${{ steps.setup_ccache.outputs.HOST_CCACHE_DIR }} \
+        LLVM_AIE_PROJECT_COMMIT=${{ needs.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }} \
+        MATRIX_OS=${{ matrix.OS }} \
+        PARALLEL_LEVEL=2 \
+        cibuildwheel --output-dir wheelhouse
+
+    - name: build aarch ubuntu wheel
+      if: contains(matrix.OS, 'ubuntu') && matrix.ARCH == 'aarch64'
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        
+        export APPLY_PATCHES=${{ inputs.APPLY_PATCHES == '' && 'true' || inputs.APPLY_PATCHES }}
+        ./scripts/apply_patches.sh
+        
+        CIBW_ARCHS=${{ matrix.ARCH }} \
+        CMAKE_GENERATOR=Ninja \
+        DATETIME=${{ needs.get_llvm_aie_project_commit.outputs.DATETIME }} \
+        LLVM_AIE_PROJECT_COMMIT=${{ needs.get_llvm_aie_project_commit.outputs.LLVM_AIE_PROJECT_COMMIT }} \
+        MATRIX_OS=${{ matrix.OS }} \
+        PARALLEL_LEVEL=2 \
+        PIP_FIND_LINKS="https://github.com/makslevental/llvm-aie-wheels/releases/expanded_assets/latest" \
+        pip wheel . -v -w wheelhouse
+
+    - name: Download cache from container ubuntu
+      if: contains(matrix.OS, 'ubuntu') && contains(matrix.ARCH, 'x86_64') && (success() || failure())
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        
+        ccache -s
+        HOST_CCACHE_DIR="$(ccache --get-config cache_dir)"
+        rm -rf $HOST_CCACHE_DIR
+        mv ./wheelhouse/.ccache $HOST_CCACHE_DIR
+        ls -la $HOST_CCACHE_DIR
+        ccache -s
+
+    - name: Reset datetime ccache
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        
+        ccache --print-stats
+        HOST_CCACHE_DIR="$(ccache --get-config cache_dir)"
+        find $HOST_CCACHE_DIR -exec touch -a -m -t "${{ needs.get_llvm_aie_project_commit.outputs.DATETIME }}00" {} \;
+
+    - name: Clean llvm-aie
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        
+        rm -rf llvm-aie
+        rm -rf build
+
+    - name: Docker prune
+      if: contains(matrix.OS, 'ubuntu')
+      run: |
+        docker system prune -a -f
+
+    - name: Get wheel version
+      id: get_wheel_version
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        pip install pkginfo
+        WHL=$(ls wheelhouse/llvm-aie-*whl)
+        echo "LLVM_AIE_WHEEL_VERSION=$(python -c "import pkginfo; w = pkginfo.Wheel('$WHL'); print(w.version.split('+')[0] + '+' + w.version.split('+')[1].rsplit('.', 1)[-1])")" | tee -a $GITHUB_OUTPUT
+
+    - name: rename non-windows
+      if: contains(matrix.OS, 'ubuntu') || contains(matrix.OS, 'macos')
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        
+        rename 's/cp310-cp310/py3-none/' wheelhouse/llvm-aie-*whl
+        rename 's/cp311-cp311/py3-none/' wheelhouse/llvm-aie-*whl
+        
+        if [[ ${{ matrix.OS }} == *"ubuntu"* ]] && [ x"${{ matrix.ARCH }}" == x"aarch64" ]; then
+          rename 's/x86_64/aarch64/' wheelhouse/llvm-aie-*whl
+        fi
+
+    - name: rename windows
+      if: contains(matrix.OS, 'windows')
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      run: |
+        ls wheelhouse/llvm-aie-*whl | Rename-Item -NewName {$_ -replace 'cp310-cp310', 'py3-none' }
+        ls wheelhouse/llvm-aie-*whl | Rename-Item -NewName {$_ -replace 'cp311-cp311', 'py3-none' }
+
+    - name: Build native_tools wheel
+      working-directory: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}
+      id: build_native_tools_wheel
+      run: |
+        
+        for TOOL in "llvm-tblgen" "llvm-aie-tblgen" "llvm-aie-linalg-ods-yaml-gen" "llvm-aie-pdll" "llvm-config" "FileCheck"; do
+          if [[ ${{ matrix.OS }} == *"windows"* ]]; then
+            TOOL="$TOOL.exe"
+          fi
+          unzip -j wheelhouse/llvm-aie-*whl "llvm_aie/bin/$TOOL" -d native_tools/
+        done
+        
+        if [[ ${{ matrix.OS }} == *"ubuntu"* ]]; then
+          PLAT="linux"
+        elif [[ ${{ matrix.OS }} == *"macos"* ]]; then
+          PLAT="macosx_12_0"
+        elif [[ ${{ matrix.OS }} == *"windows"* ]]; then
+          PLAT="win"
+        fi
+        
+        PLAT=${PLAT}_$(echo ${{ matrix.ARCH }} | tr '[:upper:]' '[:lower:]')
+        pushd native_tools
+
+        LLVM_AIE_WHEEL_VERSION=${{ steps.get_wheel_version.outputs.LLVM_AIE_WHEEL_VERSION }} \
+        python setup.py bdist_wheel --dist-dir ../wheelhouse --plat $PLAT
+        
+        popd
+
+    # done
+
+    - name: Upload wheels
+      if: success() || failure()
+      uses: actions/upload-artifact@v3
+      with:
+        path: ${{ steps.setup_base.outputs.WORKSPACE_ROOT }}/wheelhouse/*.whl
+        name: build_artifact
+
+  upload_distro_wheels:
+
+    if: github.event.schedule != '0 */4 * * *'
+
+    needs: build
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          # unpacks default artifact into dist/
+          # if `name: artifact` is omitted, the action will create extra parent dir
+          name: build_artifact
+          path: dist
+
+      - name: Release current commit
+        uses: ncipollo/[email protected]
+        with:
+          artifacts: "dist/*.whl,dist/*.tar.xz"
+          token: "${{ secrets.GITHUB_TOKEN }}"
+          tag: "nightly"
+          name: "nightly"
+          removeArtifacts: false
+          allowUpdates: true
+          replacesArtifacts: true
+          makeLatest: true