add hack to make sure testing session is killed

currently tests are interupted between retries using SIGTERM. This is not always
successful and results in simultaneous runs of the tests.

.github/workflows/unit-tests.yml

@@ -126,6 +126,10 @@ jobs:
           timeout_minutes: 20
           shell: bash
           command: SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh
+          on_retry_command: |
+            test_pid=$(cat /tmp/ignite_testing.pid)
+            kill -0 $tests_pid && kill -INT $tests_pid && sleep 5
+            while kill -0 $tests_pid > /dev/null 2>&1; do kill -9 $tests_pid; sleep 3; done
           new_command_on_retry: USE_LAST_FAILED=1  SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh
 
       - name: Upload coverage to Codecov

tests/common-test-functionality.sh

@@ -17,6 +17,8 @@ run_tests() {
     local world_size=0
     # Always clean up pytest.ini
     trap 'rm -f pytest.ini' RETURN
+    # write out the process id so that we can send signals...
+    echo $$ > /tmp/ignite_testing.pid
     # Parse arguments
     while [[ $# -gt 0 ]]
     do