improve retries

make logging more explicit and do not use /tmp incase it is cleared.

.github/workflows/unit-tests.yml

@@ -128,9 +128,15 @@ jobs:
           command: SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh
           on_retry_command: |
             echo sending kill signal until process group no longer exists...
-            tests_pid=$(cat /tmp/ignite_testing.pid)
-            kill -0 -- -$tests_pid && kill -INT -- -$tests_pid && sleep 5
-            while kill -0 -- -$tests_pid > /dev/null 2>&1; do kill -9 -- -$tests_pid; sleep 3; done
+            if [ -f .ignite_testing.pid ] && [ -s .ignite_testing.pid ]; then
+              tests_pid=$(cat .ignite_testing.pid)
+              kill -0 -- -$tests_pid && kill -INT -- -$tests_pid && sleep 5
+              set -x
+              while kill -0 -- -$tests_pid > /dev/null 2>&1; do kill -9 -- -$tests_pid; sleep 3; done
+            else
+              echo "File .ignite_testing.pid does not exist or is empty"
+            fi
+
           new_command_on_retry: USE_LAST_FAILED=1  SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh
 
       - name: Upload coverage to Codecov

tests/common-test-functionality.sh

@@ -18,7 +18,7 @@ run_tests() {
     # Always clean up pytest.ini
     trap 'rm -f pytest.ini' RETURN
     # write out the process id so that we can send signals...
-    echo $$ > /tmp/ignite_testing.pid
+    echo $$ > .ignite_testing.pid
     # Parse arguments
     while [[ $# -gt 0 ]]
     do