Add pytorch build fallback mechanism

intel · Jan 11, 2025 · 8a819ea · 8a819ea
1 parent c765476
commit 8a819ea
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 4 deletions.
diff --git a/.github/workflows/_linux_build.yml b/.github/workflows/_linux_build.yml
@@ -41,17 +41,19 @@ on:
         description: The commit id of the torch build
         value: ${{ jobs.Torch-XPU-Build.outputs.TORCH_COMMIT_ID }}
 
-permissions: read-all
+permissions:
+  issues: write
 
 jobs:
   Torch-XPU-Build:
     if: ${{ inputs.pytorch }} != 'nightly_wheel'
     runs-on: ${{ inputs.runner }}
     outputs:
-      whl_name: ${{ steps.build_version.outputs.WHL_NAME }}
       TORCH_COMMIT_ID: ${{ steps.build_version.outputs.TORCH_COMMIT_ID }}
     timeout-minutes: 900
     env:
+      commit_issue: 1280
+      GH_TOKEN: ${{ github.token }}
       NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
       DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
     steps:
@@ -96,10 +98,42 @@ jobs:
             cd ../pytorch
             export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
             pip install -r requirements.txt
-            WERROR=1 python setup.py bdist_wheel
+            gh issue view $commit_issue --json body -q .body
+            last_commit=`gh issue view $commit_issue --json body -q .body | grep ${{ inputs.pytorch }} | cut -d/ -f 7`
+            current_commit=$(git rev-parse HEAD)
+            echo ">>>>>>>>>>>>branch: ${{ inputs.pytorch }}, last commit: ${last_commit}, current commit: ${current_commit}"
+
+            WERROR=1 python setup.py bdist_wheel 2>&1 | tee pytorch_${current_commit}_build.log
+
+            if [ -f dist/torch*.whl ]; then
+              echo "Wheel build successful, update last commit in the issue https://github.com/intel/torch-xpu-ops/issues/1280"
+              cd ${{ github.workspace }}
+              gh issue view $commit_issue --json body -q .body | sed "s;${last_commit};${current_commit};" > new_body.txt
+              gh issue edit $commit_issue --body-file new_body.txt
+              cd -
+            else
+              echo "Wheel build failed, use last commit in the issue https://github.com/intel/torch-xpu-ops/issues/1280"
+              cd ${{ github.workspace }}
+              # gh issue comment $commit_issue -b "Wheel build failed with commit https://github.com/pytorch/pytorch/tree/${current_commit}. CC @chuanqi129 @mengfei25"
+              cd -
+              git clean -df .
+              git checkout $last_commit
+              # apply PRs for stock pytorch
+              python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py
+              git status && git show -s
+              git submodule sync && git submodule update --init --recursive
+              if [[ ${{ inputs.keep_torch_xpu_ops }} == 'true' ]]; then
+                echo "Don't replace torch-xpu-ops!"
+              else
+                rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/
+                # Workaround for torch-xpu-ops ci test
+                sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt
+              fi
+              WERROR=1 python setup.py bdist_wheel
+            fi
             pip install --force-reinstall dist/*.whl
             cp dist/*.whl ${{ github.workspace }}/
-            git clone https://github.com/pytorch/vision && cd vision && python setup.py install && cd ..
+            cp pytorch_${current_commit}_build.log ${{ github.workspace }}/
           else
             pip install torch torchvision torchaudio --pre --index-url https://download.pytorch.org/whl/nightly/xpu
             TORCH_COMMIT_ID=$(python -c 'import torch; print(torch.version.git_version)')
@@ -142,3 +176,9 @@ jobs:
         with:
           name: Torch-XPU-Wheel-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }} 
           path: ${{ github.workspace }}/torch*.whl
+      - name: Upload Build Log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Torch-XPU-Build-Log-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }}
+          path: ${{ github.workspace }}/pytorch_*.log
diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml
@@ -23,8 +23,11 @@ jobs:
   preci-linux-build:
     # Don't run on forked repos and draft PRs
     if: ${{ (github.repository_owner == 'intel') && (github.event.pull_request.draft == false) }}
+    permissions:
+      issues: write
     uses: ./.github/workflows/_linux_build.yml
     with:
+      pytorch: main
       runner: pvc_e2e
 
   preci-ut:
@@ -165,8 +168,11 @@ jobs:
   preci-linux-build-abi-0:
     # Don't run on forked repos and draft PRs
     if: ${{ (github.repository_owner == 'intel') && (github.event.pull_request.draft == false) }}
+    permissions:
+      issues: write
     uses: ./.github/workflows/_linux_build.yml
     with:
+      pytorch: main
       abi: 0
       runner: pvc_e2e