Skip to content

Commit

Permalink
Unit test passing for hexahedron
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnbowen42
johnbowen42 committed Feb 8, 2024
1 parent 162052a commit f35d677
Show file tree
Hide file tree
Showing 20 changed files with 917 additions and 335 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@
*.orig
__pycache__/
view
*.cache*
13 changes: 9 additions & 4 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,17 @@ set(ENABLE_MPI ON CACHE BOOL "")


if (ENABLE_CUDA)
add_compile_definitions(USE_CUDA)
set(SERAC_USE_CUDA ON CACHE BOOL "")
#add_compile_definitions(USE_CUDA)
set(ENABLE_CLANG_CUDA ON CACHE BOOL "")
set(SERAC_USE_CUDA ON CACHE BOOL "")
set(RAJA_ENABLE_CUDA ON CACHE BOOL "")
set(AXOM_USE_CUDA ON CACHE BOOL "")
set(MFEM_USE_CUDA ON CACHE BOOL "")
endif()


if (NOT MPI_C_COMPILER OR NOT MPI_CXX_COMPILER)
message(FATAL_ERROR
message(FATAL_ERROR
"Serac requires MPI. It is required to provide the MPI C/C++ "
"compiler wrappers via the CMake variables, "
"MPI_C_COMPILER and MPI_CXX_COMPILER.")
Expand Down Expand Up @@ -141,6 +142,10 @@ include(${PROJECT_SOURCE_DIR}/cmake/thirdparty/SetupSeracThirdParty.cmake)

set(CMAKE_C_FLAGS ${SERAC_CMAKE_C_FLAGS})
set(CMAKE_CXX_FLAGS ${SERAC_CMAKE_CXX_FLAGS})
string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -gdwarf-4 -fgpu-rdc" CACHE STRING "")
#set(CMAKE_CXX_FLAGS_DEBUG "-fopenmp -gdwarf-4 -fgpu-rdc" CACHE STRING "")
#set(CMAKE_CUDA_RESOLVE_DEVICE_SYMBOLS ON CACHE BOOL "")

include(${PROJECT_SOURCE_DIR}/cmake/SeracConfigHeader.cmake)

Expand Down Expand Up @@ -206,7 +211,7 @@ if (SERAC_ENABLE_CODEVELOP)
)
endif()

install(EXPORT serac-targets
install(EXPORT serac-targets
NAMESPACE serac::
DESTINATION lib/cmake
)
32 changes: 16 additions & 16 deletions cmake/thirdparty/SetupSeracThirdParty.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
# Manually set includes as system includes
foreach(_target cuda_runtime cuda)
get_target_property(_dirs ${_target} INTERFACE_INCLUDE_DIRECTORIES)
set_property(TARGET ${_target}
set_property(TARGET ${_target}
APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
"${_dirs}")
endforeach()
Expand Down Expand Up @@ -59,7 +59,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
#------------------------------------------------------------------------------
if(UMPIRE_DIR)
serac_assert_is_directory(VARIABLE_NAME UMPIRE_DIR)
find_package(umpire REQUIRED NO_DEFAULT_PATH
find_package(umpire REQUIRED NO_DEFAULT_PATH
PATHS ${UMPIRE_DIR})
message(STATUS "Umpire support is ON")
set(UMPIRE_FOUND TRUE)
Expand All @@ -73,7 +73,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
#------------------------------------------------------------------------------
if(RAJA_DIR)
serac_assert_is_directory(VARIABLE_NAME RAJA_DIR)
find_package(RAJA REQUIRED NO_DEFAULT_PATH
find_package(RAJA REQUIRED NO_DEFAULT_PATH
PATHS ${RAJA_DIR})
message(STATUS "RAJA support is ON")
set(RAJA_FOUND TRUE)
Expand All @@ -100,7 +100,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)

# Manually set includes as system includes
get_target_property(_dirs conduit::conduit INTERFACE_INCLUDE_DIRECTORIES)
set_property(TARGET conduit::conduit
set_property(TARGET conduit::conduit
APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
"${_dirs}")

Expand Down Expand Up @@ -154,10 +154,10 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
message(FATAL_ERROR "Serac+Caliper+CUDA requires CMake > 3.17.")
else()
find_package(CUDAToolkit REQUIRED)
endif()
endif()
endif()

find_package(caliper REQUIRED NO_DEFAULT_PATH
find_package(caliper REQUIRED NO_DEFAULT_PATH
PATHS ${CALIPER_DIR})
message(STATUS "Caliper support is ON")
set(CALIPER_FOUND TRUE)
Expand All @@ -177,7 +177,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)

#### Store Data that MFEM clears
set(tpls_to_save AMGX AXOM CALIPER CAMP CONDUIT HDF5
HYPRE LUA METIS MFEM NETCDF PARMETIS PETSC RAJA
HYPRE LUA METIS MFEM NETCDF PARMETIS PETSC RAJA
SUPERLU_DIST STRUMPACK SUNDIALS TRIBOL UMPIRE)
foreach(_tpl ${tpls_to_save})
set(${_tpl}_DIR_SAVE "${${_tpl}_DIR}")
Expand Down Expand Up @@ -271,7 +271,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
else()
add_subdirectory(${PROJECT_SOURCE_DIR}/mfem ${CMAKE_BINARY_DIR}/mfem)
endif()

set(MFEM_FOUND TRUE CACHE BOOL "" FORCE)

# Temporary hack to inject the hdf5_hl after netcdf and before hdf5
Expand Down Expand Up @@ -304,7 +304,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
serac_assert_is_directory(VARIABLE_NAME AXOM_DIR)

find_package(axom REQUIRED
NO_DEFAULT_PATH
NO_DEFAULT_PATH
PATHS ${AXOM_DIR}/lib/cmake)

message(STATUS "Axom support is ON")
Expand Down Expand Up @@ -420,10 +420,10 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)

# Mark the axom includes as "system" and filter unallowed directories
get_target_property(_dirs core INTERFACE_INCLUDE_DIRECTORIES)
set_property(TARGET core
set_property(TARGET core
PROPERTY INTERFACE_INCLUDE_DIRECTORIES
"${_dirs}")
set_property(TARGET core
set_property(TARGET core
APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
"${_dirs}")
else()
Expand All @@ -432,10 +432,10 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
# Mark the axom includes as "system" and filter unallowed directories
get_target_property(_dirs axom INTERFACE_INCLUDE_DIRECTORIES)
list(REMOVE_ITEM _dirs ${PROJECT_SOURCE_DIR})
set_property(TARGET axom
set_property(TARGET axom
PROPERTY INTERFACE_INCLUDE_DIRECTORIES
"${_dirs}")
set_property(TARGET axom
set_property(TARGET axom
APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
"${_dirs}")
endif()
Expand All @@ -449,7 +449,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
serac_assert_is_directory(VARIABLE_NAME TRIBOL_DIR)

find_package(tribol REQUIRED
NO_DEFAULT_PATH
NO_DEFAULT_PATH
PATHS ${TRIBOL_DIR}/lib/cmake)

if(TARGET tribol)
Expand All @@ -468,7 +468,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
else()
set(TRIBOL_FOUND OFF)
endif()

message(STATUS "Tribol support is " ${TRIBOL_FOUND})

#------------------------------------------------------------------------------
Expand Down Expand Up @@ -498,7 +498,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
# This flag is empty due to us not enabling fortran but we need to strip it
# so it doesn't propagate in our project
if("${OpenMP_Fortran_FLAGS}" STREQUAL "")
set(OpenMP_Fortran_FLAGS "$<$<NOT:$<COMPILE_LANGUAGE:Fortran>>:-fopenmp=libomp>;$<$<COMPILE_LANGUAGE:Fortran>:-fopenmp>")
set(OpenMP_Fortran_FLAGS "-fopenmp")#"$<$<NOT:$<COMPILE_LANGUAGE:Fortran>>:-fopenmp=libomp>;$<$<COMPILE_LANGUAGE:Fortran>:-fopenmp>")
endif()

foreach(_target axom)
Expand Down
19 changes: 13 additions & 6 deletions host-configs/[email protected]
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,15 @@ if(DEFINED ENV{SPACK_CC})
else()

set(CMAKE_C_COMPILER "/usr/tce/packages/clang/clang-ibm-10.0.1-gcc-8.3.1/bin/clang" CACHE PATH "")
#set(CMAKE_C_COMPILER "/usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang" CACHE PATH "")

set(CMAKE_CXX_COMPILER "/usr/tce/packages/clang/clang-ibm-10.0.1-gcc-8.3.1/bin/clang++" CACHE PATH "")
#set(CMAKE_CXX_COMPILER "/usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++" CACHE PATH "")

set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran" CACHE PATH "")

endif()

#-fopenmp -gdwarf-4 -fgpu-rdc
set(CMAKE_C_STANDARD_LIBRARIES "-lgfortran" CACHE STRING "")

set(CMAKE_CXX_STANDARD_LIBRARIES "-lgfortran" CACHE STRING "")
Expand Down Expand Up @@ -65,29 +67,34 @@ set(BLT_MPI_COMMAND_APPEND "mpibind" CACHE STRING "")
# Cuda
#------------------------------------------------

set(CUDAToolkit_ROOT "/usr/tce/packages/cuda/cuda-11.2.0" CACHE PATH "")
set(CUDAToolkit_ROOT "/usr/tce/packages/cuda/cuda-12.0.0" CACHE PATH "")
#set(CUDAToolkit_ROOT "/usr/tce/packages/cuda/cuda-10.1.105" CACHE PATH "")

set(CMAKE_CUDA_COMPILER "${CUDAToolkit_ROOT}/bin/nvcc" CACHE PATH "")

set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "")

set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.2.0" CACHE PATH "")
set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-12.0.0" CACHE PATH "")
#set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-10.1.105" CACHE PATH "")

set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")

set(ENABLE_OPENMP ON CACHE BOOL "")

set(ENABLE_CUDA ON CACHE BOOL "")

set(CMAKE_CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "")
set(ENABLE_CLANG_CUDA OFF CACHE BOOL "")

set(CMAKE_CUDA_FLAGS " --expt-extended-lambda --expt-relaxed-constexpr " CACHE STRING "")

set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
set(CMAKE_CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "")

#set(CMAKE_CUDA_FLAGS " --expt-extended-lambda --expt-relaxed-constexpr " CACHE STRING "")
#set(CMAKE_CUDA_FLAGS "-fopenmp" CACHE STRING "")

# nvcc does not like gtest's 'pthreads' flag

set(gtest_disable_pthreads ON CACHE BOOL "")
set(gtest_disable_pthreads OFF CACHE BOOL "")

set(BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3" CACHE STRING "")

Expand Down
66 changes: 33 additions & 33 deletions src/serac/numerics/functional/boundary_integral_kernels.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,41 +182,41 @@ void evaluation_kernel_impl(trial_element_type trial_elements, test_element, con

#if defined(USE_CUDA)
std::cout << "USING CUDA\n";
using policy = RAJA::cuda_exec<512>;
using policy = RAJA::cuda_exec<32>;
#else
using policy = RAJA::simd_exec;
#endif
// for each element in the domain
RAJA::forall<policy>(
RAJA::TypedRangeSegment<uint32_t>(0, num_elements),
[J, x, qf, u, rule, r, qpts_per_elem, qf_derivatives] RAJA_HOST_DEVICE(uint32_t e) {
// load the jacobians and positions for each quadrature point in this element
auto J_e = J[e];
auto x_e = x[e];
// Avoid unused warning/error ([[maybe_unused]] is not possible in the capture list)
(void)qf_derivatives;
(void)qpts_per_elem;

static constexpr trial_element_type empty_trial_element{};
// batch-calculate values / derivatives of each trial space, at each quadrature point
[[maybe_unused]] tuple qf_inputs = {promote_each_to_dual_when<indices == differentiation_index>(
get<indices>(empty_trial_element).interpolate(get<indices>(u)[e], rule))...};

// (batch) evalute the q-function at each quadrature point
auto qf_outputs = batch_apply_qf(qf, x_e, J_e, get<indices>(qf_inputs)...);

// write out the q-function derivatives after applying the
// physical_to_parent transformation, so that those transformations
// won't need to be applied in the action_of_gradient and element_gradient kernels
if constexpr (differentiation_index != serac::NO_DIFFERENTIATION) {
for (int q = 0; q < leading_dimension(qf_outputs); q++) {
qf_derivatives[e * uint32_t(qpts_per_elem) + uint32_t(q)] = get_gradient(qf_outputs[q]);
}
}

// (batch) integrate the material response against the test-space basis functions
test_element::integrate(get_value(qf_outputs), rule, &r[e]);
});
// RAJA::forall<policy>(
// RAJA::TypedRangeSegment<uint32_t>(0, num_elements),
// [J, x, qf, u, rule, r, qpts_per_elem, qf_derivatives] RAJA_HOST_DEVICE(uint32_t e) {
// // load the jacobians and positions for each quadrature point in this element
// auto J_e = J[e];
// auto x_e = x[e];
// // Avoid unused warning/error ([[maybe_unused]] is not possible in the capture list)
// (void)qf_derivatives;
// (void)qpts_per_elem;
//
// static constexpr trial_element_type empty_trial_element{};
// // batch-calculate values / derivatives of each trial space, at each quadrature point
// [[maybe_unused]] tuple qf_inputs = {promote_each_to_dual_when<indices == differentiation_index>(
// get<indices>(empty_trial_element).interpolate(get<indices>(u)[e], rule))...};
//
// // (batch) evalute the q-function at each quadrature point
// auto qf_outputs = batch_apply_qf(qf, x_e, J_e, get<indices>(qf_inputs)...);
//
// // write out the q-function derivatives after applying the
// // physical_to_parent transformation, so that those transformations
// // won't need to be applied in the action_of_gradient and element_gradient kernels
// if constexpr (differentiation_index != serac::NO_DIFFERENTIATION) {
// for (int q = 0; q < leading_dimension(qf_outputs); q++) {
// qf_derivatives[e * uint32_t(qpts_per_elem) + uint32_t(q)] = get_gradient(qf_outputs[q]);
// }
// }
//
// // (batch) integrate the material response against the test-space basis functions
// test_element::integrate(get_value(qf_outputs), rule, &r[e]);
// });
}

//clang-format off
Expand Down Expand Up @@ -278,7 +278,7 @@ void action_of_gradient_kernel(const double* dU, double* dR, derivatives_type* q
constexpr TensorProductQuadratureRule<Q> rule{};

#if defined(USE_CUDA)
using policy = RAJA::cuda_exec<512>;
using policy = RAJA::cuda_exec<32>;
#else
using policy = RAJA::simd_exec;
#endif
Expand Down Expand Up @@ -334,7 +334,7 @@ void element_gradient_kernel(ExecArrayView<double, 3, ExecutionSpace::CPU> dK,

#if defined(USE_CUDA)
std::cout << "USING CUDA :)\n";
using policy = RAJA::cuda_exec<512>;
using policy = RAJA::cuda_exec<32>;
#else
using policy = RAJA::simd_exec;
#endif
Expand Down
Loading

0 comments on commit f35d677

Please sign in to comment.