Add experimental CMake script for sharding tests in binaries

docs/cmake-integration.md

@@ -63,17 +63,19 @@ target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
 
 ## Automatic test registration
 
-Catch2's repository also contains two CMake scripts that help users
+Catch2's repository also contains three CMake scripts that help users
 with automatically registering their `TEST_CASE`s with CTest. They
 can be found in the `extras` folder, and are
 
 1) `Catch.cmake` (and its dependency `CatchAddTests.cmake`)
 2) `ParseAndAddCatchTests.cmake` (deprecated)
+3) `CatchShardTests.cmake` (and its dependency `CatchShardTestsImpl.cmake`)
 
 If Catch2 has been installed in system, both of these can be used after
 doing `find_package(Catch2 REQUIRED)`. Otherwise you need to add them
 to your CMake module path.
 
+<a id="catch_discover_tests"></a>
 ### `Catch.cmake` and `CatchAddTests.cmake`
 
 `Catch.cmake` provides function `catch_discover_tests` to get tests from
@@ -257,6 +259,49 @@ unset(OptionalCatchTestLauncher)
 ParseAndAddCatchTests(bar)
 ```
 
+
+### `CatchShardTests.cmake`
+
+> `CatchShardTests.cmake` was introduced in Catch2 X.Y.Z.
+
+`CatchShardTests.cmake` provides a function
+`catch_add_sharded_tests(TEST_BINARY)` that splits tests from `TEST_BINARY`
+into multiple shards. The tests in each shard and their order is randomized,
+and the seed changes every invocation of CTest.
+
+Currently there are 3 customization points for this script:
+
+ * SHARD_COUNT - number of shards to split target's tests into
+ * REPORTER    - reporter spec to use for tests
+ * TEST_SPEC   - test spec used for filtering tests
+
+Example usage:
+
+```
+include(CatchShardTests)
+
+catch_add_sharded_tests(foo-tests
+  SHARD_COUNT 4
+  REPORTER "xml::out=-"
+  TEST_SPEC "A"
+)
+
+catch_add_sharded_tests(tests
+  SHARD_COUNT 8
+  REPORTER "xml::out=-"
+  TEST_SPEC "B"
+)
+```
+
+This registers total of 12 CTest tests (4 + 8 shards) to run shards
+from `foo-tests` test binary, filtered by a test spec.
+
+_Note that this script is currently a proof-of-concept for reseeding
+shards per CTest run, and thus does not support (nor does it currently
+aim to support) all customization points from
+[`catch_discover_tests`](#catch_discover_tests)._
+
+
 ## CMake project options
 
 Catch2's CMake project also provides some options for other projects

extras/CatchShardTests.cmake

@@ -0,0 +1,66 @@
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+# Supported optional args:
+#  * SHARD_COUNT - number of shards to split target's tests into
+#  * REPORTER    - reporter spec to use for tests
+#  * TEST_SPEC   - test spec used for filtering tests
+function(catch_add_sharded_tests TARGET)
+  if (${CMAKE_VERSION} VERSION_LESS "3.10.0")
+    message(FATAL_ERROR "add_sharded_catch_tests only supports CMake versions 3.10.0 and up")
+  endif()
+
+  cmake_parse_arguments(
+    ""
+    ""
+    "SHARD_COUNT;REPORTER;TEST_SPEC"
+    ""
+    ${ARGN}
+  )
+
+  if (NOT DEFINED _SHARD_COUNT)
+    set(_SHARD_COUNT 2)
+  endif()
+
+  # Generate a unique name based on the extra arguments
+  string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX} ${_SHARD_COUNT}")
+  string(SUBSTRING ${args_hash} 0 7 args_hash)
+
+  set(ctest_include_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-sharded-tests-include-${args_hash}.cmake")
+  set(ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-sharded-tests-impl-${args_hash}.cmake")
+
+  file(WRITE "${ctest_include_file}"
+    "if(EXISTS \"${ctest_tests_file}\")\n"
+    "  include(\"${ctest_tests_file}\")\n"
+    "else()\n"
+    "  add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n"
+    "endif()\n"
+  )
+
+  set_property(DIRECTORY
+    APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
+  )
+
+  set(shard_impl_script_file "${CMAKE_CURRENT_LIST_DIR}/CatchShardTestsImpl.cmake")
+
+  add_custom_command(
+    TARGET ${TARGET} POST_BUILD
+    BYPRODUCTS "${ctest_tests_file}"
+    COMMAND "${CMAKE_COMMAND}"
+            -D "TARGET_NAME=${TARGET}"
+            -D "TEST_BINARY=$<TARGET_FILE:${TARGET}>"
+            -D "CTEST_FILE=${ctest_tests_file}"
+            -D "SHARD_COUNT=${_SHARD_COUNT}"
+            -D "REPORTER_SPEC=${_REPORTER}"
+            -D "TEST_SPEC=${_TEST_SPEC}"
+            -P "${shard_impl_script_file}"
+    VERBATIM
+  )
+
+
+endfunction()

extras/CatchShardTestsImpl.cmake

@@ -0,0 +1,52 @@
+
+#              Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+#   (See accompanying file LICENSE_1_0.txt or copy at
+#        https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+# Indirection for CatchShardTests that allows us to delay the script
+# file generation until build time.
+
+# Expected args:
+#  * TEST_BINARY - full path to the test binary to run sharded
+#  * CTEST_FILE  - full path to ctest script file to write to
+#  * TARGET_NAME - name of the target to shard (used for test names)
+#  * SHARD_COUNT - number of shards to split the binary into
+# Optional args:
+#  * REPORTER_SPEC - reporter specs to be passed down to the binary
+#  * TEST_SPEC     - test spec to pass down to the test binary
+
+if(NOT EXISTS "${TEST_BINARY}")
+  message(FATAL_ERROR
+    "Specified test binary '${TEST_BINARY}' does not exist"
+  )
+endif()
+
+set(other_args "")
+if (TEST_SPEC)
+  set(other_args "${other_args} ${TEST_SPEC}")
+endif()
+if (REPORTER_SPEC)
+  set(other_args "${other_args} --reporter ${REPORTER_SPEC}")
+endif()
+
+# foreach RANGE in cmake is inclusive of the end, so we have to adjust it
+math(EXPR adjusted_shard_count "${SHARD_COUNT} - 1")
+
+file(WRITE "${CTEST_FILE}"
+  "string(RANDOM LENGTH 8 ALPHABET \"0123456789abcdef\" rng_seed)\n"
+  "\n"
+  "foreach(shard_idx RANGE ${adjusted_shard_count})\n"
+  "  add_test(${TARGET_NAME}-shard-" [[${shard_idx}]] "/${adjusted_shard_count}\n"
+  "    ${TEST_BINARY}"
+  " --shard-index " [[${shard_idx}]]
+  " --shard-count ${SHARD_COUNT}"
+  " --rng-seed " [[0x${rng_seed}]]
+  " --order rand"
+  "${other_args}"
+  "\n"
+  "  )\n"
+  "endforeach()\n"
+)