-
Notifications
You must be signed in to change notification settings - Fork 88
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Enable CTest Resources #1373
Enable CTest Resources #1373
Changes from 38 commits
ae0bb9d
6e336ee
258e329
6ca783f
846bc92
96eb18f
5bf2034
e2f6496
aa144ea
907b359
8174beb
6870df1
8df1afd
b991e1a
da69ea6
45e9db4
514599d
b326ee7
c8692ab
89f95b4
2ea4003
89906c8
e6c845b
5525494
d265e44
e760d24
cada2ce
1602d73
b84728e
d05da30
959618a
5675cee
4c2defa
8fad906
6ff4c3e
30b91bb
6c59ec2
65061cb
d454a84
4150292
498123f
9071cfb
6e4bc33
95240dc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,26 +1,19 @@ | ||
set(gko_test_single_args "MPI_SIZE") | ||
set(gko_test_resource_args "RESOURCE_LOCAL_CORES;RESOURCE_TYPE") | ||
set(gko_test_single_args "MPI_SIZE;${gko_test_resource_args}") | ||
upsj marked this conversation as resolved.
Show resolved
Hide resolved
|
||
set(gko_test_multi_args "DISABLE_EXECUTORS;ADDITIONAL_LIBRARIES;ADDITIONAL_INCLUDES") | ||
set(gko_test_option_args "NO_RESOURCES") | ||
|
||
## Replaces / by _ to create valid target names from relative paths | ||
function(ginkgo_build_test_name test_name target_name) | ||
file(RELATIVE_PATH REL_BINARY_DIR | ||
${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}) | ||
${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}) | ||
string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This file contains some formatting changes. We should really find a way to get a consistent cmake formatting. |
||
set(${target_name} ${TEST_TARGET_NAME} PARENT_SCOPE) | ||
endfunction(ginkgo_build_test_name) | ||
|
||
function(ginkgo_create_gtest_mpi_main) | ||
add_library(gtest_mpi_main "") | ||
target_sources(gtest_mpi_main | ||
PRIVATE | ||
${PROJECT_SOURCE_DIR}/core/test/mpi/gtest/mpi_listener.cpp) | ||
find_package(MPI 3.1 COMPONENTS CXX REQUIRED) | ||
target_link_libraries(gtest_mpi_main PRIVATE GTest::GTest MPI::MPI_CXX) | ||
endfunction(ginkgo_create_gtest_mpi_main) | ||
endfunction() | ||
|
||
## Set up shared target properties and handle ADDITIONAL_LIBRARIES/ADDITIONAL_INCLUDES | ||
## `MPI_SIZE size` causes the tests to be run with `size` MPI processes. | ||
function(ginkgo_set_test_target_properties test_target_name) | ||
function(ginkgo_set_test_target_properties test_target_name test_library_suffix) | ||
cmake_parse_arguments(PARSE_ARGV 1 set_properties "" "${gko_test_single_args}" "${gko_test_multi_args}") | ||
if (GINKGO_FAST_TESTS) | ||
target_compile_definitions(${test_target_name} PRIVATE GINKGO_FAST_TESTS) | ||
|
@@ -31,26 +24,62 @@ function(ginkgo_set_test_target_properties test_target_name) | |
if (GINKGO_COMPILING_DPCPP_TEST AND GINKGO_DPCPP_SINGLE_MODE) | ||
target_compile_definitions(${test_target_name} PRIVATE GINKGO_DPCPP_SINGLE_MODE=1) | ||
endif() | ||
if (GINKGO_CHECK_CIRCULAR_DEPS) | ||
if(GINKGO_CHECK_CIRCULAR_DEPS) | ||
target_link_libraries(${test_target_name} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}") | ||
endif() | ||
if (set_properties_MPI_SIZE) | ||
if(NOT TARGET gtest_mpi_main) | ||
ginkgo_create_gtest_mpi_main() | ||
endif() | ||
set(gtest_main gtest_mpi_main MPI::MPI_CXX) | ||
if(set_properties_MPI_SIZE) | ||
target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main_mpi${test_library_suffix}) | ||
else() | ||
set(gtest_main GTest::Main) | ||
target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main${test_library_suffix}) | ||
endif() | ||
target_compile_features(${test_target_name} PUBLIC cxx_std_14) | ||
target_compile_options(${test_target_name} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_COMPILER_FLAGS}>) | ||
target_include_directories(${test_target_name} PRIVATE ${Ginkgo_BINARY_DIR} ${set_properties_ADDITIONAL_INCLUDES}) | ||
target_link_libraries(${test_target_name} PRIVATE ginkgo ${gtest_main} GTest::GTest ${set_properties_ADDITIONAL_LIBRARIES}) | ||
target_link_libraries(${test_target_name} PRIVATE ginkgo GTest::GTest ${set_properties_ADDITIONAL_LIBRARIES}) | ||
endfunction() | ||
|
||
function(ginkgo_add_resource_requirement test_name) | ||
cmake_parse_arguments(PARSE_ARGV 1 add_rr "${gko_test_option_args}" "${gko_test_single_args}" "") | ||
if(add_rr_NO_RESOURCES) | ||
return() | ||
endif() | ||
|
||
if (NOT add_rr_RESOURCE_TYPE) | ||
message(FATAL_ERROR "Need to provide resource type used by test.") | ||
endif () | ||
|
||
if(add_rr_RESOURCE_TYPE STREQUAL "ref") | ||
set(single_resource "cpu:1") | ||
elseif(add_rr_RESOURCE_TYPE STREQUAL "cpu") | ||
if(NOT add_rr_RESOURCE_LOCAL_CORES) | ||
set(add_rr_RESOURCE_LOCAL_CORES ${GINKGO_CI_TEST_OMP_PARALLELISM}) | ||
endif() | ||
if(NOT add_rr_RESOURCE_LOCAL_CORES MATCHES "^[0-9]+") | ||
message(FATAL_ERROR "Resource specification is invalid: RESOURCE_LOCAL_CORES=${add_rr_RESOURCE_LOCAL_CORES}") | ||
endif() | ||
|
||
set(single_resource "cpu:${add_rr_RESOURCE_LOCAL_CORES}") | ||
elseif(add_rr_RESOURCE_TYPE MATCHES "^(cudagpu|hipgpu|sycl)$") | ||
set(single_resource "${add_rr_RESOURCE_TYPE}:1") | ||
upsj marked this conversation as resolved.
Show resolved
Hide resolved
|
||
else() | ||
message(FATAL_ERROR "Unrecognized resource type ${add_rr_RESOURCE_TYPE}, allowed are: ref, cpu, cudagpu, hipgpu, sycl.") | ||
endif() | ||
upsj marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
if(NOT add_rr_MPI_SIZE) | ||
set(add_rr_MPI_SIZE 1) | ||
endif() | ||
set_property(TEST ${test_name} | ||
PROPERTY | ||
RESOURCE_GROUPS "${add_rr_MPI_SIZE},${single_resource}") | ||
endfunction() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. when using gpu executor, we do not have limitation on cpu side. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We could add an additional CPU restriction, but since all tests compare against sequential reference, which only uses a single core, and it's unlikely that a system has more GPUs than cores, I think it shouldn't make a difference. |
||
|
||
|
||
## Adds a test to the list executed by ctest and sets its output binary name | ||
## Possible additional arguments: | ||
## - `MPI_SIZE size` causes the tests to be run with `size` MPI processes. | ||
## - `RESOURCE_LOCAL_CORES` the number of threads used by a test, default is | ||
## $GINKGO_CI_TEST_OMP_PARALLELISM | ||
## - `RESOURCE_TYPE` the resource type, can be ref, cpu, cudagpu, hipgpu, sycl | ||
## - `DISABLE_EXECUTORS exec1 exec2` disables the test for certain backends (if built for multiple) | ||
## - `ADDITIONAL_LIBRARIES lib1 lib2` adds additional target link dependencies | ||
## - `ADDITIONAL_INCLUDES path1 path2` adds additional target include paths | ||
|
@@ -71,6 +100,11 @@ function(ginkgo_add_test test_name test_target_name) | |
COMMAND ${test_target_name} | ||
WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>") | ||
endif() | ||
# use custom target `tests` to build only test binaries | ||
add_dependencies(tests ${test_target_name}) | ||
|
||
ginkgo_add_resource_requirement(${REL_BINARY_DIR}/${test_name} ${ARGN}) | ||
|
||
set(test_preload) | ||
if (GINKGO_TEST_NONDEFAULT_STREAM AND GINKGO_BUILD_CUDA) | ||
set(test_preload $<TARGET_FILE:identify_stream_usage_cuda>:${test_preload}) | ||
|
@@ -87,9 +121,9 @@ endfunction() | |
function(ginkgo_create_test test_name) | ||
ginkgo_build_test_name(${test_name} test_target_name) | ||
add_executable(${test_target_name} ${test_name}.cpp) | ||
target_link_libraries(${test_target_name} PRIVATE ${create_test_ADDITIONAL_LIBRARIES}) | ||
ginkgo_set_test_target_properties(${test_target_name} ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN}) | ||
target_link_libraries(${test_target_name}) | ||
ginkgo_set_test_target_properties(${test_target_name} "" ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE ref) | ||
upsj marked this conversation as resolved.
Show resolved
Hide resolved
|
||
endfunction(ginkgo_create_test) | ||
|
||
## Test compiled with dpcpp | ||
|
@@ -99,8 +133,8 @@ function(ginkgo_create_dpcpp_test test_name) | |
target_compile_features(${test_target_name} PUBLIC cxx_std_17) | ||
target_compile_options(${test_target_name} PRIVATE ${GINKGO_DPCPP_FLAGS}) | ||
target_link_options(${test_target_name} PRIVATE -fsycl-device-code-split=per_kernel) | ||
ginkgo_set_test_target_properties(${test_target_name} ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN}) | ||
ginkgo_set_test_target_properties(${test_target_name} "_dpcpp" ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE sycl) | ||
# Note: MKL_ENV is empty on linux. Maybe need to apply MKL_ENV to all test. | ||
if (MKL_ENV) | ||
set_tests_properties(${test_target_name} PROPERTIES ENVIRONMENT "${MKL_ENV}") | ||
|
@@ -133,8 +167,8 @@ function(ginkgo_create_cuda_test_internal test_name filename test_target_name) | |
if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18) | ||
set_target_properties(${test_target_name} PROPERTIES CUDA_ARCHITECTURES OFF) | ||
endif() | ||
ginkgo_set_test_target_properties(${test_target_name} ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN}) | ||
ginkgo_set_test_target_properties(${test_target_name} "_cuda" ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE cudagpu) | ||
endfunction(ginkgo_create_cuda_test_internal) | ||
|
||
## Test compiled with HIP | ||
|
@@ -189,10 +223,26 @@ function(ginkgo_create_hip_test_internal test_name filename test_target_name add | |
${hiprand_INCLUDE_DIRS} | ||
${HIPSPARSE_INCLUDE_DIRS} | ||
) | ||
ginkgo_set_test_target_properties(${test_target_name} ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN}) | ||
ginkgo_set_test_target_properties(${test_target_name} "_hip" ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE hipgpu) | ||
endfunction(ginkgo_create_hip_test_internal) | ||
|
||
|
||
## Test compiled with OpenMP | ||
function(ginkgo_create_omp_test test_name) | ||
ginkgo_build_test_name(${test_name} test_target_name) | ||
ginkgo_create_omp_test_internal(${test_name} ${test_name}.cpp ${test_target_name} "" ${ARGN}) | ||
endfunction() | ||
|
||
function(ginkgo_create_omp_test_internal test_name filename test_target_name) | ||
ginkgo_build_test_name(${test_name} test_target_name) | ||
add_executable(${test_target_name} ${test_name}.cpp) | ||
target_compile_definitions(${test_target_name} PRIVATE GKO_COMPILING_OMP) | ||
target_link_libraries(${test_target_name} PRIVATE OpenMP::OpenMP_CXX) | ||
ginkgo_set_test_target_properties(${test_target_name} "_omp" ${ARGN}) | ||
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN} RESOURCE_TYPE cpu) | ||
endfunction() | ||
|
||
## Common test compiled with the host compiler, one target for each enabled backend | ||
function(ginkgo_create_common_test test_name) | ||
if(GINKGO_BUILD_OMP) | ||
|
@@ -214,20 +264,38 @@ function(ginkgo_create_common_test_internal test_name exec_type exec) | |
if(exec IN_LIST common_test_DISABLE_EXECUTORS) | ||
return() | ||
endif() | ||
if (exec STREQUAL reference) | ||
set(test_resource_type ref) | ||
elseif (exec STREQUAL omp) | ||
set(test_resource_type cpu) | ||
elseif (exec STREQUAL cuda) | ||
set(test_resource_type cudagpu) | ||
elseif (exec STREQUAL hip) | ||
set(test_resource_type hipgpu) | ||
else () | ||
set(test_resource_type sycl) | ||
endif () | ||
ginkgo_build_test_name(${test_name} test_target_name) | ||
string(TOUPPER ${exec} exec_upper) | ||
|
||
# set up actual test | ||
set(test_target_name ${test_target_name}_${exec}) | ||
add_executable(${test_target_name} ${test_name}.cpp) | ||
|
||
# also need to add runtime libraries for other backends | ||
if (exec STREQUAL omp) | ||
target_link_libraries(${test_target_name} PRIVATE OpenMP::OpenMP_CXX) | ||
endif () | ||
|
||
target_compile_definitions(${test_target_name} PRIVATE EXEC_TYPE=${exec_type} EXEC_NAMESPACE=${exec} GKO_COMPILING_${exec_upper}) | ||
target_link_libraries(${test_target_name} PRIVATE ${common_test_ADDITIONAL_LIBRARIES}) | ||
# use float for DPC++ if necessary | ||
if((exec STREQUAL "dpcpp") AND GINKGO_DPCPP_SINGLE_MODE) | ||
target_compile_definitions(${test_target_name} PRIVATE GINKGO_COMMON_SINGLE_MODE=1) | ||
target_compile_definitions(${test_target_name} PRIVATE GINKGO_DPCPP_SINGLE_MODE=1) | ||
endif() | ||
ginkgo_set_test_target_properties(${test_target_name} ${ARGN}) | ||
ginkgo_add_test(${test_name}_${exec} ${test_target_name} ${ARGN}) | ||
ginkgo_set_test_target_properties(${test_target_name} "_${exec}" ${ARGN}) | ||
ginkgo_add_test(${test_name}_${exec} ${test_target_name} ${ARGN} RESOURCE_TYPE ${test_resource_type}) | ||
endfunction(ginkgo_create_common_test_internal) | ||
|
||
## Common test compiled with the device compiler, one target for each enabled backend | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
function(add_gtest_main suffix definitions) | ||
add_library(ginkgo_gtest_main${suffix} STATIC ginkgo_main.cpp resources.cpp) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. any reason for STATIC? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Windows seems to have issues with global variables being declared in a shared library, but referenced from another library. This seems the easiest fix |
||
target_link_libraries(ginkgo_gtest_main${suffix} PUBLIC Ginkgo::ginkgo GTest::GTest) | ||
target_compile_definitions(ginkgo_gtest_main${suffix} PRIVATE ${definitions}) | ||
ginkgo_compile_features(ginkgo_gtest_main${suffix}) | ||
if (GINKGO_BUILD_MPI) | ||
add_library(ginkgo_gtest_main_mpi${suffix} STATIC ginkgo_mpi_main.cpp resources.cpp) | ||
target_link_libraries(ginkgo_gtest_main_mpi${suffix} PUBLIC Ginkgo::ginkgo GTest::GTest MPI::MPI_CXX) | ||
target_compile_definitions(ginkgo_gtest_main_mpi${suffix} PRIVATE ${definitions}) | ||
ginkgo_compile_features(ginkgo_gtest_main_mpi${suffix}) | ||
endif() | ||
endfunction() | ||
|
||
add_gtest_main("" "") | ||
add_library(ginkgo_gtest_main_reference ALIAS ginkgo_gtest_main) | ||
if (GINKGO_BUILD_MPI) | ||
add_library(ginkgo_gtest_main_mpi_reference ALIAS ginkgo_gtest_main_mpi) | ||
endif() | ||
if (GINKGO_BUILD_OMP) | ||
add_gtest_main("_omp" "GKO_COMPILING_OMP") | ||
endif() | ||
if (GINKGO_BUILD_CUDA) | ||
add_gtest_main("_cuda" "GKO_COMPILING_CUDA") | ||
endif() | ||
if (GINKGO_BUILD_HIP) | ||
add_gtest_main("_hip" "GKO_COMPILING_HIP") | ||
endif() | ||
if (GINKGO_BUILD_DPCPP) | ||
add_gtest_main("_dpcpp" "GKO_COMPILING_DPCPP") | ||
endif() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Are these changes from the PR #1394 ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
no, I just noticed we are barely using nla1, and this enables more jobs to be run there.