CI: find and print backtraces (#5424)

I think this should work to find and print backtraces after all CI tests
have run.

This is for Azure only. Local backtraces still need be inspected
manually.

After trying many solutions, the current strategy is:
- avoid removing the backtrace files in the `cleanup` step of each test
(we continue to remove all other files);
- have a separate workflow step to find and print the backtrace files in
the Azure job (this is executed always).

The new Azure workflow step is labeled "Logs" and it comes right after
the step labeled "Test".

.azure-pipelines.yml

@@ -77,7 +77,7 @@ jobs:
     displayName: Cache Python Libraries
 
   - bash: |
-      set -eu -o pipefail
+      set -o nounset errexit pipefail
       cat /proc/cpuinfo | grep "model name" | sort -u
       df -h
       echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
@@ -146,25 +146,36 @@ jobs:
     displayName: 'Install dependencies'
 
   - bash: |
-      set -eu -o pipefail
+      # set options
+      set -o nounset errexit pipefail
+      # display disk space usage
       df -h
-
       # configure
       export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON"
       cmake -S . -B build       \
         ${AMReX_CMAKE_FLAGS}    \
         ${WARPX_CMAKE_FLAGS}    \
         -DWarpX_TEST_CLEANUP=ON \
         -DWarpX_TEST_FPETRAP=ON
-
       # build
       cmake --build build -j 2
+      # display disk space usage
       df -h
     displayName: 'Build'
 
   - bash: |
-      set -eu -o pipefail
-
+      # set options
+      set -o nounset errexit pipefail
       # run tests (exclude pytest.AMReX when running Python tests)
       ctest --test-dir build --output-on-failure -E AMReX
     displayName: 'Test'
+
+  - bash: |
+      # set options
+      set -o nounset errexit pipefail
+      # find and print backtrace
+      find build/bin/ -type f -name "Backtrace*" \
+        -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \
+        -exec cat {} \;
+    displayName: 'Logs'
+    condition: always()

Examples/CMakeLists.txt

@@ -198,7 +198,7 @@ function(add_warpx_test
     if(WarpX_TEST_CLEANUP)
         add_test(
             NAME ${name}.cleanup
-            COMMAND ${CMAKE_COMMAND} -E rm -rf ${THIS_WORKING_DIR}
+            COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/Examples/test_cleanup.cmake ${THIS_WORKING_DIR}
         )
         # test cleanup depends on test run
         set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.run")

Examples/test_cleanup.cmake

@@ -0,0 +1,7 @@
+# delete all test files except backtrace
+file(GLOB test_files ${CMAKE_ARGV3}/*)
+foreach(file ${test_files})
+    if(NOT ${file} MATCHES "Backtrace*")
+        execute_process(COMMAND ${CMAKE_COMMAND} -E rm -r ${file})
+    endif()
+endforeach()