Add support for Kokkos Remote Spaces 1.2, rebases on top of master

CMakeLists.txt

@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 3.16)
+
+project(ExaMiniMD LANGUAGES CXX)
+
+include(GNUInstallDirs)
+
+find_package(Kokkos REQUIRED)
+
+option(ENABLE_MPI "Whether to build with MPI" OFF)
+option(ENABLE_KOKKOS_REMOTE_SPACES "Whether to build with Kokkos Remote Spaces" OFF)
+
+if (ENABLE_MPI)
+  find_package(MPI REQUIRED CXX)
+  message(STATUS "Building with MPI support")
+  set(BACKEND_NAME MPI)
+  list(APPEND BACKENDS ${BACKEND_NAME})
+endif()
+
+if (ENABLE_KOKKOS_REMOTE_SPACES)
+  find_package(MPI REQUIRED CXX)
+  find_package(KokkosRemoteSpaces REQUIRED)
+  message(STATUS "Building with Kokkos Remote Spaces support")
+  set(BACKEND_NAME KokkosRemoteSpaces)
+  list(APPEND BACKENDS ${BACKEND_NAME})
+endif()
+
+list(LENGTH BACKENDS N_BACKENDS)
+if (${N_BACKENDS} GREATER_EQUAL "2")
+  message(FATAL_ERROR "Must specify a single valid communiction implementation. ${N_BACKENDS} given")
+endif()
+
+if (${N_BACKENDS} EQUAL "0")
+  message(STATUS "Building with serial comm type")
+endif()
+
+
+add_subdirectory(src)
+add_subdirectory(input)

input/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(FILES
+  in.lj)
+
+foreach(FILE ${FILES})
+  configure_file(${FILE} ${FILE} COPYONLY)
+endforeach()
+
+add_subdirectory(snap)

input/in.lj

@@ -1,11 +1,11 @@
-# 3d Lennard-Jones melt
+# Example using Lennard-Jones potential
 
 units		lj
 atom_style	atomic
 
 newton off
 lattice		fcc 0.8442
-region		box block 0 40 0 40 0 40
+region		box block 0 100 0 100 0 100
 create_box	1 box
 create_atoms	1 box
 mass		1 2.0
@@ -16,8 +16,8 @@ pair_style	lj/cut 2.5
 pair_coeff	1 1 1.0 1.0 2.5
 
 neighbor	0.3 bin
-neigh_modify delay 0 every 20 check no
+neigh_modify	every 20 one 50
 fix		1 all nve
 thermo 10
 
-run		100
+run	        100

input/snap/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(FILES
+  in.snap.Ta06A
+  in.snap.W
+  Ta06A.snap
+  Ta06A.snapcoeff
+  Ta06A.snapparam
+  W.snapcoeff
+  W.snapparam)
+
+foreach(FILE ${FILES})
+  configure_file(${FILE} ${FILE} COPYONLY)
+endforeach()

input/snap/in.snap.W

@@ -8,7 +8,7 @@ newton 		on
 # generate the box and atom positions using a BCC lattice
 
 lattice         sc 3.1803
-region		box block 0 4 0 8 0 8
+region		box block 0 8 0 8 0 8
 create_box	1 box
 create_atoms	1 box
 
@@ -27,7 +27,7 @@ thermo		10
 # Set up NVE run
 
 neighbor 1.0 bin
-neigh_modify once no every 1 delay 0 check yes
+neigh_modify once no every 20 delay 0 check yes
 
 # Run MD
 

src/CMakeLists.txt

@@ -0,0 +1,21 @@
+FILE(GLOB SRCS *.cpp)
+add_executable(ExaMiniMD ${SRCS})
+
+set(SUBDIRECTORIES
+  binning_types
+  comm_types
+  force_types
+  neighbor_types)
+
+foreach(SUBDIR ${SUBDIRECTORIES})
+  add_subdirectory(${SUBDIR})
+endforeach()
+
+if(ENABLE_MPI)
+   target_compile_definitions(ExaMiniMD PRIVATE EXAMINIMD_ENABLE_MPI)
+endif()
+
+target_include_directories(ExaMiniMD PRIVATE ${Kokkos_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${SUBDIRECTORIES})
+target_link_libraries(ExaMiniMD PRIVATE $<$<BOOL:${ENABLE_MPI}>:MPI::MPI_CXX> Kokkos::kokkos $<$<BOOL:${ENABLE_KOKKOS_REMOTE_SPACES}>:Kokkos::kokkosremotespaces>)
+
+install(TARGETS ExaMiniMD DESTINATION ${CMAKE_INSTALL_BINDIR})

src/binning_types/CMakeLists.txt

@@ -0,0 +1,2 @@
+FILE(GLOB SRCS *.cpp)
+target_sources(ExaMiniMD PRIVATE ${SRCS})

src/binning_types/binning_kksort.cpp

@@ -71,7 +71,7 @@ namespace {
 void BinningKKSort::create_binning(T_X_FLOAT dx_in, T_X_FLOAT dy_in, T_X_FLOAT dz_in, int halo_depth, bool do_local, bool do_ghost, bool sort) {
   if(do_local||do_ghost) {
     nhalo = halo_depth;
-    std::pair<T_INT,T_INT> range(do_local?0:system->N_local,
+    Kokkos::pair<T_INT,T_INT> range(do_local?0:system->N_local,
                                  do_ghost?system->N_local+system->N_ghost:system->N_local);
 
     nbinx = T_INT(system->sub_domain_x/dx_in);

src/comm_lib.cpp

@@ -0,0 +1,88 @@
+//************************************************************************
+//  ExaMiniMD v. 1.0
+//  Copyright (2018) National Technology & Engineering Solutions of Sandia,
+//  LLC (NTESS).
+//
+//  Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
+//  retains certain rights in this software.
+//
+//  ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
+//  use in source and binary forms, with or without modification, are
+//  permitted provided that the following conditions are met:
+//
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//
+//    2. Redistributions in binary form must reproduce the above copyright notice,
+//       this list of conditions and the following disclaimer in the documentation
+//       and/or other materials provided with the distribution.
+//
+//    3. Neither the name of the Corporation nor the names of the contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
+//  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+//  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+//  IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+//  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+//  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+//  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+//  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+//  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  POSSIBILITY OF SUCH DAMAGE.
+//
+//  Questions? Contact Christian R. Trott ([email protected])
+//************************************************************************
+
+#include <comm_lib.h>
+#include <assert.h>
+
+#if EXAMINIMD_ENABLE_MPI
+#include <mpi.h>
+#endif
+
+#ifdef EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES
+#include <Kokkos_RemoteSpaces.hpp>
+#endif
+
+void comm_lib_init(int argc, char* argv[]) {
+#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
+  int mpi_thread_level_available;
+  int mpi_thread_level_required = MPI_THREAD_MULTIPLE;
+
+#ifdef KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_SERIAL
+  mpi_thread_level_required = MPI_THREAD_SINGLE;
+#endif
+
+  MPI_Init_thread(&argc, &argv, mpi_thread_level_required,
+                  &mpi_thread_level_available);
+  assert(mpi_thread_level_available >= mpi_thread_level_required);
+
+#ifdef KRS_ENABLE_SHMEMSPACE
+  shmem_init_thread(mpi_thread_level_required, &mpi_thread_level_available);
+  assert(mpi_thread_level_available >= mpi_thread_level_required);
+#endif
+
+#ifdef KRS_ENABLE_NVSHMEMSPACE
+  MPI_Comm mpi_comm;
+  nvshmemx_init_attr_t attr;
+  mpi_comm      = MPI_COMM_WORLD;
+  attr.mpi_comm = &mpi_comm;
+  nvshmemx_init_attr(NVSHMEMX_INIT_WITH_MPI_COMM, &attr);
+#endif
+}
+
+void comm_lib_finalize() {
+#if defined (EXAMINIMD_ENABLE_MPI) || defined (EXAMINIMD_ENABLE_KOKKOS_REMOTE_SPACES)
+#ifdef KRS_ENABLE_SHMEMSPACE
+  shmem_finalize();
+#endif
+#ifdef KRS_ENABLE_NVSHMEMSPACE
+  nvshmem_finalize();
+#endif
+  MPI_Finalize();
+#endif
+#endif
+}

src/comm_lib.h

@@ -0,0 +1,47 @@
+//************************************************************************
+//  ExaMiniMD v. 1.0
+//  Copyright (2018) National Technology & Engineering Solutions of Sandia,
+//  LLC (NTESS).
+//
+//  Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government
+//  retains certain rights in this software.
+//
+//  ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and
+//  use in source and binary forms, with or without modification, are
+//  permitted provided that the following conditions are met:
+//
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//
+//    2. Redistributions in binary form must reproduce the above copyright notice,
+//       this list of conditions and the following disclaimer in the documentation
+//       and/or other materials provided with the distribution.
+//
+//    3. Neither the name of the Corporation nor the names of the contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED
+//  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+//  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+//  IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+//  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+//  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+//  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+//  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+//  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//  POSSIBILITY OF SUCH DAMAGE.
+//
+//  Questions? Contact Christian R. Trott ([email protected])
+//************************************************************************
+
+#pragma once
+
+#ifndef COMM_INIT_H
+#define COMM_INIT_H
+
+void comm_lib_init(int argc, char* argv[]);
+void comm_lib_finalize();
+
+#endif

src/comm_types/CMakeLists.txt

@@ -0,0 +1,3 @@
+FILE(GLOB SRCS *.cpp)
+target_sources(ExaMiniMD PRIVATE ${SRCS})
+target_sources(ExaMiniMD PRIVATE ${SRCS})