expand use of on_retry_command

also adds retry for gpu tests
pytorch · May 10, 2024 · 426d4a3 · 426d4a3
1 parent 6b5fbeb
commit 426d4a3
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 14 deletions.
diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml
@@ -121,18 +121,13 @@ jobs:
 
       - name: Run GPU Unit Tests
         continue-on-error: false
-        run: |
-
-          script=$(cat << EOF
-
-          set -xe
-
-          bash tests/run_gpu_tests.sh 2
-
-          EOF
-          )
-
-          docker exec -t pthd /bin/bash -c "${script}"
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 5
+          timeout_minutes: 30
+          shell: bash
+          command: docker exec -t pthd /bin/bash -xec 'tests/run_gpu_tests.sh 2'
+          new_command_on_retry: docker exec -e USE_LAST_FAILED=1 -t pthd /bin/bash -xec 'tests/run_gpu_tests.sh 2'
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/hvd-tests.yml b/.github/workflows/hvd-tests.yml
@@ -81,6 +81,15 @@ jobs:
           timeout_minutes: 25
           shell: bash
           command: bash tests/run_cpu_tests.sh
+          on_retry_command: |
+            echo sending kill signal until process group no longer exists...
+            if [ -f .ignite_testing.pid ] && [ -s .ignite_testing.pid ]; then
+              tests_pid=$(cat .ignite_testing.pid)
+              ps -p $tests_pid &> /dev/null && pkill -INT -g $tests_pid && sleep 5
+              ps -p $tests_pid &> /dev/null && pkill -KILL -g $tests_pid
+            else
+              echo "File .ignite_testing.pid does not exist or is empty"
+            fi
           new_command_on_retry: USE_LAST_FAILED=1   bash tests/run_cpu_tests.sh
 
       - name: Upload coverage to Codecov

diff --git a/.github/workflows/tpu-tests.yml b/.github/workflows/tpu-tests.yml
@@ -97,6 +97,15 @@ jobs:
           command: |
             python -c "import torch_xla; print('torch xla version:', torch_xla.__version__)"
             bash tests/run_tpu_tests.sh
+          on_retry_command: |
+            echo sending kill signal until process group no longer exists...
+            if [ -f .ignite_testing.pid ] && [ -s .ignite_testing.pid ]; then
+              tests_pid=$(cat .ignite_testing.pid)
+              ps -p $tests_pid &> /dev/null && pkill -INT -g $tests_pid && sleep 5
+              ps -p $tests_pid &> /dev/null && pkill -KILL -g $tests_pid
+            else
+              echo "File .ignite_testing.pid does not exist or is empty"
+            fi
           new_command_on_retry: USE_LAST_FAILED=1  bash tests/run_tpu_tests.sh
         env:
           LD_LIBRARY_PATH: ${{ env.LD_LIBRARY_PATH }}:${{ env.Python_ROOT_DIR }}/lib

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -130,8 +130,8 @@ jobs:
             echo sending kill signal until process group no longer exists...
             if [ -f .ignite_testing.pid ] && [ -s .ignite_testing.pid ]; then
               tests_pid=$(cat .ignite_testing.pid)
-              ps -p $tests_pid > /dev/null && pkill -INT -g $tests_pid && sleep 5
-              ps -p $tests_pid > /dev/null && pkill -KILL -g $tests_pid
+              ps -p $tests_pid &> /dev/null && pkill -INT -g $tests_pid && sleep 5
+              ps -p $tests_pid &> /dev/null && pkill -KILL -g $tests_pid
             else
               echo "File .ignite_testing.pid does not exist or is empty"
             fi