diff --git a/.jenkins/continuous.groovy b/.jenkins/continuous.groovy index b71f28e90..42fbd8617 100644 --- a/.jenkins/continuous.groovy +++ b/.jenkins/continuous.groovy @@ -89,6 +89,8 @@ pipeline { -D CMAKE_CXX_COMPILER=$KOKKOS_DIR/bin/nvcc_wrapper \ -D CMAKE_CXX_EXTENSIONS=OFF \ -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$ARBORX_DIR" \ + -D MPIEXEC_PREFLAGS="--allow-run-as-root" \ + -D MPIEXEC_MAX_NUMPROCS=4 \ examples \ ''' sh 'make VERBOSE=1' @@ -148,6 +150,8 @@ pipeline { -D CMAKE_CXX_COMPILER=$KOKKOS_DIR/bin/nvcc_wrapper \ -D CMAKE_CXX_EXTENSIONS=OFF \ -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$ARBORX_DIR" \ + -D MPIEXEC_PREFLAGS="--allow-run-as-root" \ + -D MPIEXEC_MAX_NUMPROCS=4 \ examples \ ''' sh 'make VERBOSE=1' @@ -205,6 +209,8 @@ pipeline { -D CMAKE_CXX_COMPILER=clang++ \ -D CMAKE_CXX_EXTENSIONS=OFF \ -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$ARBORX_DIR" \ + -D MPIEXEC_PREFLAGS="--allow-run-as-root" \ + -D MPIEXEC_MAX_NUMPROCS=4 \ examples \ ''' sh 'make VERBOSE=1' @@ -266,6 +272,8 @@ pipeline { -D CMAKE_CXX_COMPILER=clang++ \ -D CMAKE_CXX_EXTENSIONS=OFF \ -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$ARBORX_DIR" \ + -D MPIEXEC_PREFLAGS="--allow-run-as-root" \ + -D MPIEXEC_MAX_NUMPROCS=4 \ examples \ ''' sh 'make VERBOSE=1' @@ -329,6 +337,8 @@ pipeline { -D CMAKE_CXX_EXTENSIONS=OFF \ -D CMAKE_BUILD_TYPE=RelWithDebInfo \ -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$ARBORX_DIR" \ + -D MPIEXEC_PREFLAGS="--allow-run-as-root" \ + -D MPIEXEC_MAX_NUMPROCS=4 \ examples \ ''' sh 'make VERBOSE=1' @@ -391,6 +401,8 @@ pipeline { -D CMAKE_CXX_FLAGS="-Wno-unknown-cuda-version" \ -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$ARBORX_DIR;$ONE_DPL_DIR" \ -D ONEDPL_PAR_BACKEND=serial \ + -D MPIEXEC_PREFLAGS="--allow-run-as-root" \ + -D MPIEXEC_MAX_NUMPROCS=4 \ examples \ ''' sh 'make VERBOSE=1' diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 4fe135477..39b6e1f34 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -13,6 +13,10 @@ add_subdirectory(simple_intersection) add_subdirectory(molecular_dynamics) +if(ARBORX_ENABLE_MPI) + add_subdirectory(distributed_tree) +endif() + find_package(Boost COMPONENTS program_options) if(Boost_FOUND) add_subdirectory(viz) diff --git a/examples/distributed_tree/CMakeLists.txt b/examples/distributed_tree/CMakeLists.txt new file mode 100644 index 000000000..8ecad2be9 --- /dev/null +++ b/examples/distributed_tree/CMakeLists.txt @@ -0,0 +1,11 @@ +add_executable(ArborX_DistributedTree_KNN.exe distributed_knn.cpp) +target_link_libraries(ArborX_DistributedTree_KNN.exe ArborX::ArborX) +add_test(NAME ArborX_DistributedTree_KNN_Example COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ./ArborX_DistributedTree_KNN.exe ${MPIEXEC_POSTFLAGS}) + +add_executable(ArborX_DistributedTree_Intersects.exe distributed_intersects.cpp) +target_link_libraries(ArborX_DistributedTree_Intersects.exe ArborX::ArborX) +add_test(NAME ArborX_DistributedTree_Intersects_Example COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ./ArborX_DistributedTree_Intersects.exe ${MPIEXEC_POSTFLAGS}) + +add_executable(ArborX_DistributedTree_IntersectsCallback.exe distributed_intersects_callback.cpp) +target_link_libraries(ArborX_DistributedTree_IntersectsCallback.exe ArborX::ArborX) +add_test(NAME ArborX_DistributedTree_IntersectsCallback_Example COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ./ArborX_DistributedTree_IntersectsCallback.exe ${MPIEXEC_POSTFLAGS}) diff --git a/examples/distributed_tree/distributed_intersects.cpp b/examples/distributed_tree/distributed_intersects.cpp new file mode 100644 index 000000000..3f9576ed1 --- /dev/null +++ b/examples/distributed_tree/distributed_intersects.cpp @@ -0,0 +1,89 @@ +/**************************************************************************** + * Copyright (c) 2017-2022 by the ArborX authors * + * All rights reserved. * + * * + * This file is part of the ArborX library. ArborX is * + * distributed under a BSD 3-clause license. For the licensing terms see * + * the LICENSE file in the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#include + +#include + +#include "distributed_setup.hpp" +#include + +using ExecutionSpace = Kokkos::DefaultExecutionSpace; +using MemorySpace = ExecutionSpace::memory_space; + +namespace Example +{ +template +struct Intersects +{ + Points points; + float radius; +}; +template +Intersects(Points const &, int) -> Intersects; + +struct IndexAndRank +{ + int index; + int rank; +}; + +} // namespace Example + +template +struct ArborX::AccessTraits, ArborX::PredicatesTag> +{ + static KOKKOS_FUNCTION std::size_t size(Example::Intersects const &x) + { + return x.points.extent(0); + } + static KOKKOS_FUNCTION auto get(Example::Intersects const &x, + std::size_t i) + { + return ArborX::intersects(ArborX::Sphere(x.points(i), x.radius)); + } + using memory_space = MemorySpace; +}; + +int main(int argc, char *argv[]) +{ + MPI_Init(&argc, &argv); + Kokkos::initialize(argc, argv); + { + MPI_Comm comm = MPI_COMM_WORLD; + ExecutionSpace exec; + Kokkos::View points_device; + ArborX::DistributedTree tree = + ArborXExample::create_tree(comm, exec, points_device); + + Kokkos::View values("values", 0); + Kokkos::View offsets("offsets", 0); + tree.query(exec, Example::Intersects{points_device, 1.}, values, offsets); + + auto host_values = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, values); + auto host_offsets = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, offsets); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + for (unsigned int i = 0; i + 1 < host_offsets.size(); ++i) + { + std::cout << "Results for query " << i << " on MPI rank " << comm_rank + << '\n'; + for (int j = host_offsets(i); j < host_offsets(i + 1); ++j) + std::cout << "point " << host_values(j).index << ", rank " + << host_values(j).rank << std::endl; + } + } + Kokkos::finalize(); + MPI_Finalize(); + return 0; +} diff --git a/examples/distributed_tree/distributed_intersects_callback.cpp b/examples/distributed_tree/distributed_intersects_callback.cpp new file mode 100644 index 000000000..9c46e481c --- /dev/null +++ b/examples/distributed_tree/distributed_intersects_callback.cpp @@ -0,0 +1,121 @@ +/**************************************************************************** + * Copyright (c) 2017-2022 by the ArborX authors * + * All rights reserved. * + * * + * This file is part of the ArborX library. ArborX is * + * distributed under a BSD 3-clause license. For the licensing terms see * + * the LICENSE file in the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#include + +#include + +#include "distributed_setup.hpp" +#include + +using ExecutionSpace = Kokkos::DefaultExecutionSpace; +using MemorySpace = ExecutionSpace::memory_space; + +namespace Example +{ +template +struct Intersects +{ + Points points; + float radius; + int mpi_rank; +}; +template +Intersects(Points const &, float, int) -> Intersects; + +struct IndexAndRank +{ + int index; + int rank; +}; + +template +struct InlinePrintCallback +{ + Kokkos::View points; + int mpi_rank; + + InlinePrintCallback(Kokkos::View const &points_, + int mpi_rank_) + : points(points_) + , mpi_rank(mpi_rank_) + {} + + template + KOKKOS_FUNCTION void operator()(Predicate const &predicate, + int primitive_index, + OutputFunctor const &out) const + { + auto data = ArborX::getData(predicate); + auto const &point = points(primitive_index); + printf("Intersection for query %d from MPI rank %d on MPI rank %d for " + "point %f,%f,%f with index %d\n", + data.index, data.rank, mpi_rank, point[0], point[1], point[2], + primitive_index); + + out({primitive_index, mpi_rank}); + } +}; + +} // namespace Example + +template +struct ArborX::AccessTraits, ArborX::PredicatesTag> +{ + static KOKKOS_FUNCTION std::size_t size(Example::Intersects const &x) + { + return x.points.extent(0); + } + static KOKKOS_FUNCTION auto get(Example::Intersects const &x, int i) + { + return attach(ArborX::intersects(ArborX::Sphere(x.points(i), x.radius)), + Example::IndexAndRank{i, x.mpi_rank}); + } + using memory_space = MemorySpace; +}; + +int main(int argc, char *argv[]) +{ + MPI_Init(&argc, &argv); + Kokkos::initialize(argc, argv); + { + MPI_Comm comm = MPI_COMM_WORLD; + ExecutionSpace exec; + Kokkos::View points_device; + ArborX::DistributedTree tree = + ArborXExample::create_tree(comm, exec, points_device); + + Kokkos::View values("values", 0); + Kokkos::View offsets("offsets", 0); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + tree.query( + exec, Example::Intersects{points_device, 1., comm_rank}, + Example::InlinePrintCallback(points_device, comm_rank), + values, offsets); + + auto host_values = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, values); + auto host_offsets = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, offsets); + for (unsigned int i = 0; i + 1 < host_offsets.size(); ++i) + { + std::cout << "Results for query " << i << " on MPI rank " << comm_rank + << '\n'; + for (int j = host_offsets(i); j < host_offsets(i + 1); ++j) + std::cout << "point " << host_values(j).index << ", rank " + << host_values(j).rank << std::endl; + } + } + Kokkos::finalize(); + MPI_Finalize(); + return 0; +} diff --git a/examples/distributed_tree/distributed_knn.cpp b/examples/distributed_tree/distributed_knn.cpp new file mode 100644 index 000000000..9497e3c47 --- /dev/null +++ b/examples/distributed_tree/distributed_knn.cpp @@ -0,0 +1,89 @@ +/**************************************************************************** + * Copyright (c) 2017-2022 by the ArborX authors * + * All rights reserved. * + * * + * This file is part of the ArborX library. ArborX is * + * distributed under a BSD 3-clause license. For the licensing terms see * + * the LICENSE file in the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#include + +#include + +#include "distributed_setup.hpp" +#include + +using ExecutionSpace = Kokkos::DefaultExecutionSpace; +using MemorySpace = ExecutionSpace::memory_space; + +namespace Example +{ +template +struct Nearest +{ + Points points; + int k; +}; +template +Nearest(Points const &, int) -> Nearest; + +struct IndexAndRank +{ + int index; + int rank; +}; + +} // namespace Example + +template +struct ArborX::AccessTraits, ArborX::PredicatesTag> +{ + static KOKKOS_FUNCTION std::size_t size(Example::Nearest const &x) + { + return x.points.extent(0); + } + static KOKKOS_FUNCTION auto get(Example::Nearest const &x, + std::size_t i) + { + return ArborX::nearest(x.points(i), x.k); + } + using memory_space = MemorySpace; +}; + +int main(int argc, char *argv[]) +{ + MPI_Init(&argc, &argv); + Kokkos::initialize(argc, argv); + { + MPI_Comm comm = MPI_COMM_WORLD; + ExecutionSpace exec; + Kokkos::View points_device; + ArborX::DistributedTree tree = + ArborXExample::create_tree(comm, exec, points_device); + + Kokkos::View values("values", 0); + Kokkos::View offsets("offsets", 0); + tree.query(exec, Example::Nearest{points_device, 3}, values, offsets); + + auto host_values = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, values); + auto host_offsets = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, offsets); + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + for (unsigned int i = 0; i + 1 < host_offsets.size(); ++i) + { + std::cout << "Results for query " << i << " on MPI rank " << comm_rank + << '\n'; + for (int j = host_offsets(i); j < host_offsets(i + 1); ++j) + std::cout << "point " << host_values(j).index << ", rank " + << host_values(j).rank << std::endl; + } + } + Kokkos::finalize(); + MPI_Finalize(); + return 0; +} diff --git a/examples/distributed_tree/distributed_setup.hpp b/examples/distributed_tree/distributed_setup.hpp new file mode 100644 index 000000000..5c44354cd --- /dev/null +++ b/examples/distributed_tree/distributed_setup.hpp @@ -0,0 +1,45 @@ +/**************************************************************************** + * Copyright (c) 2017-2022 by the ArborX authors * + * All rights reserved. * + * * + * This file is part of the ArborX library. ArborX is * + * distributed under a BSD 3-clause license. For the licensing terms see * + * the LICENSE file in the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#include + +#include + +#include + +namespace ArborXExample +{ + +template +ArborX::DistributedTree +create_tree(MPI_Comm comm, ExecutionSpace const &exec, + Kokkos::View &points_device) +{ + int comm_rank; + MPI_Comm_rank(comm, &comm_rank); + int comm_size; + MPI_Comm_size(comm, &comm_size); + ArborX::Point lower_left_corner = {static_cast(comm_rank), + static_cast(comm_rank), + static_cast(comm_rank)}; + ArborX::Point center = {static_cast(comm_rank) + .5f, + static_cast(comm_rank) + .5f, + static_cast(comm_rank) + .5f}; + std::vector points = {lower_left_corner, center}; + points_device = Kokkos::create_mirror_view_and_copy( + MemorySpace{}, + Kokkos::View( + points.data(), points.size())); + + ArborX::DistributedTree tree(comm, exec, points_device); + return tree; +} +} // namespace ArborXExample